diff --git a/cmd/gateway.go b/cmd/gateway.go index 2fc593d5..d078a1a0 100644 --- a/cmd/gateway.go +++ b/cmd/gateway.go @@ -20,6 +20,8 @@ import ( "github.com/nextlevelbuilder/goclaw/internal/eventbus" kg "github.com/nextlevelbuilder/goclaw/internal/knowledgegraph" "github.com/nextlevelbuilder/goclaw/internal/channels/discord" + "github.com/nextlevelbuilder/goclaw/internal/channels/facebook" + "github.com/nextlevelbuilder/goclaw/internal/channels/pancake" "github.com/nextlevelbuilder/goclaw/internal/channels/feishu" slackchannel "github.com/nextlevelbuilder/goclaw/internal/channels/slack" "github.com/nextlevelbuilder/goclaw/internal/channels/telegram" @@ -179,17 +181,16 @@ func runGateway() { } setupMemoryEmbeddings(pgStores, providerRegistry) + // Resolve background provider for consolidation + vault enrichment. + // Fallback: background.provider → agent.default_provider → first registered provider. + bgProvider, bgModel := resolveBackgroundProvider(cfg, providerRegistry) + // V3: Wire consolidation pipeline (episodic → semantic → KG → dreaming) if pgStores.Episodic != nil { - var consolidationProvider providers.Provider - if names := providerRegistry.ListForTenant(providers.MasterTenantID); len(names) > 0 { - consolidationProvider, _ = providerRegistry.GetForTenant(providers.MasterTenantID, names[0]) - } - if consolidationProvider != nil { - // Create KG extractor for semantic worker (entity/relation extraction from episodic summaries) + if bgProvider != nil { var kgExtractor *kg.Extractor if pgStores.KnowledgeGraph != nil { - kgExtractor = kg.NewExtractor(consolidationProvider, consolidationProvider.DefaultModel(), 0) + kgExtractor = kg.NewExtractor(bgProvider, bgModel, 0) } cleanupConsolidation := consolidation.Register(consolidation.ConsolidationDeps{ EpisodicStore: pgStores.Episodic, @@ -197,34 +198,31 @@ func runGateway() { KGStore: pgStores.KnowledgeGraph, SessionStore: pgStores.Sessions, EventBus: domainBus, - Provider: consolidationProvider, - Model: consolidationProvider.DefaultModel(), + Provider: bgProvider, + Model: bgModel, Extractor: kgExtractor, + AgentStore: pgStores.Agents, }) defer cleanupConsolidation() - slog.Info("consolidation pipeline registered") + slog.Info("consolidation pipeline registered", "provider", bgProvider.Name(), "model", bgModel) } else { slog.Warn("consolidation pipeline skipped: no provider available") } } // V3: Wire vault enrichment worker (async summary + embedding + auto-linking). - // Resolves provider independently from consolidation pipeline. - if pgStores.Vault != nil { - var vaultProvider providers.Provider - if names := providerRegistry.ListForTenant(providers.MasterTenantID); len(names) > 0 { - vaultProvider, _ = providerRegistry.GetForTenant(providers.MasterTenantID, names[0]) - } - if vaultProvider != nil { - cleanupVaultEnrich := vault.RegisterEnrichWorker(vault.EnrichWorkerDeps{ - VaultStore: pgStores.Vault, - Provider: vaultProvider, - Model: vaultProvider.DefaultModel(), - EventBus: domainBus, - }) - defer cleanupVaultEnrich() - slog.Info("vault enrichment worker registered") - } + var enrichProgress *vault.EnrichProgress + if pgStores.Vault != nil && bgProvider != nil { + cleanupVaultEnrich, ep := vault.RegisterEnrichWorker(vault.EnrichWorkerDeps{ + VaultStore: pgStores.Vault, + Provider: bgProvider, + Model: bgModel, + EventBus: domainBus, + MsgBus: msgBus, + }) + enrichProgress = ep + defer cleanupVaultEnrich() + slog.Info("vault enrichment worker registered", "provider", bgProvider.Name(), "model", bgModel) } loadBootstrapFiles(pgStores, workspace, agentCfg) @@ -282,7 +280,7 @@ func runGateway() { var mcpPool *mcpbridge.Pool var mediaStore *media.Store var postTurn tools.PostTurnProcessor - contextFileInterceptor, mcpPool, mediaStore, postTurn = wireExtras(pgStores, agentRouter, providerRegistry, msgBus, pgStores.Sessions, toolsReg, toolPE, skillsLoader, hasMemory, traceCollector, workspace, cfg.Gateway.InjectionAction, cfg, sandboxMgr, redisClient, domainBus) + contextFileInterceptor, mcpPool, mediaStore, postTurn = wireExtras(pgStores, agentRouter, providerRegistry, modelReg, msgBus, pgStores.Sessions, toolsReg, toolPE, skillsLoader, hasMemory, traceCollector, workspace, cfg.Gateway.InjectionAction, cfg, sandboxMgr, redisClient, domainBus) if mcpPool != nil { defer mcpPool.Stop() } @@ -297,8 +295,10 @@ func runGateway() { agentRouter: agentRouter, toolsReg: toolsReg, skillsLoader: skillsLoader, + enrichProgress: enrichProgress, workspace: workspace, dataDir: dataDir, + domainBus: domainBus, } gatewayAddr := loopbackAddr(cfg.Gateway.Host, cfg.Gateway.Port) @@ -314,7 +314,11 @@ func runGateway() { // Wire dependencies for system prompt preview parity. if agentsH != nil { agentsH.SetPreviewDeps(toolsReg, skillsLoader) - agentsH.SetPreviewStores(pgStores.Teams, pgStores.AgentLinks) + var skillAccess store.SkillAccessStore + if pgStores.Skills != nil { + skillAccess, _ = pgStores.Skills.(store.SkillAccessStore) + } + agentsH.SetPreviewStores(pgStores.Teams, pgStores.AgentLinks, skillAccess) } // External wake/trigger API @@ -417,6 +421,8 @@ func runGateway() { instanceLoader.RegisterFactory(channels.TypeZaloPersonal, zalopersonal.FactoryWithPendingStore(pgStores.PendingMessages)) instanceLoader.RegisterFactory(channels.TypeWhatsApp, whatsapp.FactoryWithDB(pgStores.DB, pgStores.PendingMessages, "pgx")) instanceLoader.RegisterFactory(channels.TypeSlack, slackchannel.FactoryWithPendingStore(pgStores.PendingMessages)) + instanceLoader.RegisterFactory(channels.TypeFacebook, facebook.Factory) + instanceLoader.RegisterFactory(channels.TypePancake, pancake.Factory) if err := instanceLoader.LoadAll(context.Background()); err != nil { slog.Error("failed to load channel instances from DB", "error", err) } @@ -507,8 +513,10 @@ func runGateway() { methods.NewTenantsMethods(pgStores.Tenants, msgBus, workspace).Register(server.Router()) server.SetTenantsHandler(httpapi.NewTenantsHandler(pgStores.Tenants, msgBus, workspace)) server.Router().SetTenantStore(pgStores.Tenants) - // Permission cache for tenant membership checks + // Permission cache for tenant membership checks. Store on deps so + // lifecycle shutdown can call Close() to stop the sweep goroutines. permCache := cache.NewPermissionCache() + deps.permCache = permCache msgBus.Subscribe("permission-cache", func(e bus.Event) { if p, ok := e.Payload.(bus.CacheInvalidatePayload); ok { permCache.HandleInvalidation(p) @@ -534,3 +542,39 @@ func runGateway() { sigCh: sigCh, }) } + +// resolveBackgroundProvider picks the LLM provider+model for background workers +// (vault enrichment, consolidation). Fallback chain: +// +// background.provider/model → agent.default_provider/model → first registered provider. +func resolveBackgroundProvider(cfg *config.Config, reg *providers.Registry) (providers.Provider, string) { + try := func(name, model string) (providers.Provider, string, bool) { + if name == "" { + return nil, "", false + } + p, err := reg.GetForTenant(providers.MasterTenantID, name) + if err != nil || p == nil { + return nil, "", false + } + if model == "" { + model = p.DefaultModel() + } + return p, model, true + } + + // 1. Explicit background config + if p, m, ok := try(cfg.Gateway.BackgroundProvider, cfg.Gateway.BackgroundModel); ok { + return p, m + } + // 2. Agent default provider + if p, m, ok := try(cfg.Agents.Defaults.Provider, cfg.Agents.Defaults.Model); ok { + return p, m + } + // 3. First registered provider (legacy fallback) + if names := reg.ListForTenant(providers.MasterTenantID); len(names) > 0 { + if p, m, ok := try(names[0], ""); ok { + return p, m + } + } + return nil, "" +} diff --git a/cmd/gateway_callbacks.go b/cmd/gateway_callbacks.go index 13615d84..f4f4dcfc 100644 --- a/cmd/gateway_callbacks.go +++ b/cmd/gateway_callbacks.go @@ -30,8 +30,8 @@ func buildEnsureUserProfile(as store.AgentStore) agent.EnsureUserProfileFunc { // isNew=true seeds all files; isNew=false only seeds if user has zero files // (avoids re-seeding BOOTSTRAP.md after auto-cleanup on server restart). func buildSeedUserFiles(as store.AgentStore) agent.SeedUserFilesFunc { - return func(ctx context.Context, agentID uuid.UUID, userID, agentType string, isNew bool) error { - _, err := bootstrap.SeedUserFiles(ctx, as, agentID, userID, agentType, !isNew) + return func(ctx context.Context, agentID uuid.UUID, userID, agentType string, isNew bool, channelMeta *bootstrap.ChannelMeta) error { + _, err := bootstrap.SeedUserFiles(ctx, as, agentID, userID, agentType, !isNew, channelMeta) return err } } diff --git a/cmd/gateway_consumer_helpers.go b/cmd/gateway_consumer_helpers.go index 3ac407db..1ef647ee 100644 --- a/cmd/gateway_consumer_helpers.go +++ b/cmd/gateway_consumer_helpers.go @@ -91,6 +91,18 @@ func extractSessionMetadata(msg bus.InboundMessage, peerKind string) map[string] return meta } +// buildPancakeSessionLabel returns "Pancake:{senderName}:{pageName}" with non-empty parts only. +func buildPancakeSessionLabel(senderName, pageName string) string { + label := "Pancake" + if senderName != "" { + label += ":" + senderName + } + if pageName != "" { + label += ":" + pageName + } + return label +} + // buildAnnounceOutMeta builds outbound metadata for announce messages so that // Send() can route replies to the correct forum topic or DM thread. func buildAnnounceOutMeta(localKey string) map[string]string { diff --git a/cmd/gateway_consumer_normal.go b/cmd/gateway_consumer_normal.go index 05f1f60b..aa1609a0 100644 --- a/cmd/gateway_consumer_normal.go +++ b/cmd/gateway_consumer_normal.go @@ -108,6 +108,12 @@ func processNormalMessage( } } + // Set session label for Pancake channels: "Pancake:{SenderName}:{PageName}" + if msg.Metadata["pancake_mode"] != "" { + label := buildPancakeSessionLabel(msg.Metadata["display_name"], msg.Metadata["page_name"]) + deps.SessStore.SetLabel(ctx, sessionKey, label) + } + // Auto-collect channel contacts for the contact selector. // Skip internal senders (system:*, notification:*, teammate:*, ticker:*, session_send_tool). if deps.ContactCollector != nil && msg.SenderID != "" && !bus.IsInternalSender(msg.SenderID) { diff --git a/cmd/gateway_cron.go b/cmd/gateway_cron.go index 8d186947..8bb9759b 100644 --- a/cmd/gateway_cron.go +++ b/cmd/gateway_cron.go @@ -78,8 +78,12 @@ func makeCronJobHandler(sched *scheduler.Scheduler, msgBus *bus.MessageBus, cfg ) } - // Build context with tenant scope so agent loop events are scoped correctly. - cronCtx := store.WithTenantID(context.Background(), job.TenantID) + // Build context with tenant scope and timeout so agent loop events are + // scoped correctly and a hung agent can't block the cron scheduler forever. + jobTimeout := cfg.Cron.JobTimeoutDuration() + cronCtx, cancelCron := context.WithTimeout(context.Background(), jobTimeout) + defer cancelCron() + cronCtx = store.WithTenantID(cronCtx, job.TenantID) // Reset session before each cron run to prevent tool errors from previous // runs from polluting the context and blocking future executions (#294). @@ -106,8 +110,13 @@ func makeCronJobHandler(sched *scheduler.Scheduler, msgBus *bus.MessageBus, cfg TraceTags: []string{"cron"}, }) - // Block until the scheduled run completes - outcome := <-outCh + // Block until the scheduled run completes or the timeout fires. + var outcome scheduler.RunOutcome + select { + case outcome = <-outCh: + case <-cronCtx.Done(): + return nil, fmt.Errorf("cron job %s timed out after %s", job.Name, jobTimeout) + } if outcome.Err != nil { return nil, outcome.Err } diff --git a/cmd/gateway_deps.go b/cmd/gateway_deps.go index 67c378cd..b05f6823 100644 --- a/cmd/gateway_deps.go +++ b/cmd/gateway_deps.go @@ -3,13 +3,16 @@ package cmd import ( "github.com/nextlevelbuilder/goclaw/internal/agent" "github.com/nextlevelbuilder/goclaw/internal/bus" + "github.com/nextlevelbuilder/goclaw/internal/cache" "github.com/nextlevelbuilder/goclaw/internal/channels" "github.com/nextlevelbuilder/goclaw/internal/config" + "github.com/nextlevelbuilder/goclaw/internal/eventbus" "github.com/nextlevelbuilder/goclaw/internal/gateway" "github.com/nextlevelbuilder/goclaw/internal/providers" "github.com/nextlevelbuilder/goclaw/internal/skills" "github.com/nextlevelbuilder/goclaw/internal/store" "github.com/nextlevelbuilder/goclaw/internal/tools" + "github.com/nextlevelbuilder/goclaw/internal/vault" ) // gatewayDeps holds shared dependencies used across the extracted gateway setup functions. @@ -24,6 +27,9 @@ type gatewayDeps struct { agentRouter *agent.Router toolsReg *tools.Registry skillsLoader *skills.Loader // optional: enables skill creation in evolution approval + permCache *cache.PermissionCache // nil if no tenant store; closed on shutdown to stop sweep goroutines + enrichProgress *vault.EnrichProgress // nil if enrichment worker not registered workspace string dataDir string + domainBus eventbus.DomainEventBus } diff --git a/cmd/gateway_evolution_cron.go b/cmd/gateway_evolution_cron.go index 38a2042e..31f6b77b 100644 --- a/cmd/gateway_evolution_cron.go +++ b/cmd/gateway_evolution_cron.go @@ -2,6 +2,7 @@ package cmd import ( "context" + "database/sql" "log/slog" "time" @@ -9,36 +10,94 @@ import ( "github.com/nextlevelbuilder/goclaw/internal/store" ) -// runEvolutionCron runs the v3 evolution suggestion engine (daily) and -// evaluation/rollback check (weekly) as background goroutines. +// evolutionCronLockID is a PG advisory lock ID for preventing duplicate cron runs. +// Only one gateway instance should run evolution analysis at a time. +const evolutionCronLockID int64 = 0x65766F6C // "evol" + +// runEvolutionCron runs the v3 evolution suggestion engine (daily at 3:00 AM) +// and evaluation/rollback check (weekly on Sundays at 3:00 AM) as a background goroutine. // Designed to be called with `go runEvolutionCron(...)`. func runEvolutionCron(stores *store.Stores, engine *agent.SuggestionEngine) { - dailyTicker := time.NewTicker(24 * time.Hour) - defer dailyTicker.Stop() - - weeklyTicker := time.NewTicker(7 * 24 * time.Hour) - defer weeklyTicker.Stop() - - // Run first analysis 1 minute after startup (warm-up). + // Wait 1 minute after startup for warm-up, then run first analysis. time.Sleep(1 * time.Minute) runSuggestionAnalysis(stores, engine) for { - select { - case <-dailyTicker.C: - runSuggestionAnalysis(stores, engine) - case <-weeklyTicker.C: + nextDaily := nextDailyRun(3) // 3:00 AM server time + + timer := time.NewTimer(time.Until(nextDaily)) + <-timer.C + timer.Stop() + + runSuggestionAnalysis(stores, engine) + + // Weekly evaluation on Sundays. + if time.Now().Weekday() == time.Sunday { runEvolutionEvaluation(stores) } } } -// runSuggestionAnalysis lists agents with evolution metrics enabled and runs analysis. +// nextDailyRun returns the next occurrence of the given hour (0-23) today or tomorrow. +// Uses AddDate to handle DST transitions correctly (calendar day, not 24h). +func nextDailyRun(hour int) time.Time { + now := time.Now() + next := time.Date(now.Year(), now.Month(), now.Day(), hour, 0, 0, 0, now.Location()) + if !now.Before(next) { + next = next.AddDate(0, 0, 1) // calendar day, not 24h — DST-safe + } + return next +} + +// tryAdvisoryLock acquires a PG session-level advisory lock on a pinned connection. +// Returns the pinned connection (caller must close) and whether the lock was acquired. +// No-op (returns nil, true) when db is nil (SQLite/desktop edition runs single-instance). +func tryAdvisoryLock(ctx context.Context, db *sql.DB) (*sql.Conn, bool) { + if db == nil { + return nil, true + } + conn, err := db.Conn(ctx) + if err != nil { + slog.Debug("evolution.cron.conn_failed", "error", err) + return nil, false + } + var acquired bool + if err := conn.QueryRowContext(ctx, "SELECT pg_try_advisory_lock($1)", evolutionCronLockID).Scan(&acquired); err != nil { + conn.Close() + slog.Debug("evolution.cron.advisory_lock_failed", "error", err) + return nil, false + } + if !acquired { + conn.Close() + return nil, false + } + return conn, true +} + +// releaseAdvisoryLock releases a PG advisory lock on the pinned connection and closes it. +func releaseAdvisoryLock(ctx context.Context, conn *sql.Conn) { + if conn == nil { + return + } + _, _ = conn.ExecContext(ctx, "SELECT pg_advisory_unlock($1)", evolutionCronLockID) + conn.Close() +} + +// runSuggestionAnalysis lists agents with evolution enabled and runs analysis. +// Note: List(ctx, "") uses bare context (no tenant) to list ALL agents cross-tenant. +// Per-agent calls are scoped via WithTenantID. func runSuggestionAnalysis(stores *store.Stores, engine *agent.SuggestionEngine) { ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) defer cancel() - // List all agents (empty ownerID = all, tenant-scoped via context). + // Acquire advisory lock on a pinned connection to prevent duplicate runs across instances. + conn, acquired := tryAdvisoryLock(ctx, stores.DB) + if !acquired { + slog.Debug("evolution.cron.skipped_lock_held") + return + } + defer releaseAdvisoryLock(ctx, conn) + agents, err := stores.Agents.List(ctx, "") if err != nil { slog.Warn("evolution.cron.list_agents_failed", "error", err) @@ -54,6 +113,9 @@ func runSuggestionAnalysis(stores *store.Stores, engine *agent.SuggestionEngine) if !flags.EvolutionMetrics { continue } + if !flags.EvolutionSuggest { + continue // metrics enabled but suggestions disabled — skip analysis + } agentCtx := store.WithTenantID(ctx, ag.TenantID) if _, err := engine.Analyze(agentCtx, ag.ID); err != nil { slog.Debug("evolution.cron.analyze_failed", "agent", ag.ID, "error", err) @@ -86,6 +148,9 @@ func runEvolutionEvaluation(stores *store.Stores) { if !flags.EvolutionMetrics { continue } + if !flags.EvolutionSuggest { + continue // skip evaluation for agents with suggestions disabled + } agentCtx := store.WithTenantID(ctx, ag.TenantID) if err := agent.EvaluateApplied(agentCtx, ag.ID, guardrails, stores.EvolutionMetrics, stores.EvolutionSuggestions, stores.Agents); err != nil { slog.Debug("evolution.cron.eval_failed", "agent", ag.ID, "error", err) diff --git a/cmd/gateway_http_wiring.go b/cmd/gateway_http_wiring.go index 2a5bea94..4048cab5 100644 --- a/cmd/gateway_http_wiring.go +++ b/cmd/gateway_http_wiring.go @@ -167,12 +167,17 @@ func (d *gatewayDeps) wireHTTPHandlersOnServer( if d.pgStores.Agents != nil { evoOpts = append(evoOpts, httpapi.WithAgentStore(d.pgStores.Agents)) } + if d.pgStores.BuiltinToolTenantCfgs != nil { + evoOpts = append(evoOpts, httpapi.WithToolTenantCfgs(d.pgStores.BuiltinToolTenantCfgs)) + } d.server.SetEvolutionHandler(httpapi.NewEvolutionHandler(d.pgStores.EvolutionMetrics, d.pgStores.EvolutionSuggestions, evoOpts...)) } // V3: Knowledge Vault document API if d.pgStores != nil && d.pgStores.Vault != nil { - d.server.SetVaultHandler(httpapi.NewVaultHandler(d.pgStores.Vault, d.pgStores.Teams)) + vh := httpapi.NewVaultHandler(d.pgStores.Vault, d.pgStores.Teams, d.workspace, d.domainBus, d.pgStores.Agents, d.pgStores.Teams) + vh.SetEnrichProgress(d.enrichProgress) + d.server.SetVaultHandler(vh) } // V3: Episodic memory summaries API diff --git a/cmd/gateway_lifecycle.go b/cmd/gateway_lifecycle.go index 8552224b..27f990d8 100644 --- a/cmd/gateway_lifecycle.go +++ b/cmd/gateway_lifecycle.go @@ -159,6 +159,11 @@ func (d *gatewayDeps) runLifecycle( // Close provider resources (e.g. Claude CLI temp files) d.providerRegistry.Close() + // Stop permission cache sweep goroutines so they don't leak past shutdown. + if d.permCache != nil { + d.permCache.Close() + } + // Stop sandbox pruning + release containers if deps.sandboxMgr != nil { deps.sandboxMgr.Stop() diff --git a/cmd/gateway_managed.go b/cmd/gateway_managed.go index cd286451..081e1cbe 100644 --- a/cmd/gateway_managed.go +++ b/cmd/gateway_managed.go @@ -42,6 +42,7 @@ func wireExtras( stores *store.Stores, agentRouter *agent.Router, providerReg *providers.Registry, + modelReg providers.ModelRegistry, msgBus *bus.MessageBus, sessStore store.SessionStore, toolsReg *tools.Registry, @@ -140,10 +141,14 @@ func wireExtras( autoInjector = memorypkg.NewAutoInjector(stores.Episodic, stores.EvolutionMetrics) } + // vaultIntc is set later by wireVault but captured by closure in OnTextUploaded. + var vaultIntc *tools.VaultInterceptor + resolver := agent.NewManagedResolver(agent.ResolverDeps{ AgentStore: stores.Agents, ProviderStore: stores.Providers, ProviderReg: providerReg, + ModelRegistry: modelReg, Bus: msgBus, Sessions: sessStore, Tools: toolsReg, @@ -157,6 +162,7 @@ func wireExtras( ContextFileLoader: contextFileLoader, BootstrapCleanup: buildBootstrapCleanup(stores.Agents), CacheInvalidate: buildCacheInvalidate(contextFileInterceptor), + DefaultTimezone: appCfg.Cron.DefaultTimezone, InjectionAction: injectionAction, MaxMessageChars: appCfg.Gateway.MaxMessageChars, CompactionCfg: appCfg.Agents.Defaults.Compaction, @@ -184,6 +190,11 @@ func wireExtras( AutoInjector: autoInjector, EvolutionMetricsStore: stores.EvolutionMetrics, DomainBus: domainBus, + OnTextUploaded: func(ctx context.Context, path, content string) { + if vaultIntc != nil { + vaultIntc.AfterWrite(ctx, path, content) + } + }, OnEvent: func(event agent.AgentEvent) { // Sign /v1/files/ and /v1/media/ URLs in content before delivery. // Sessions store clean paths; signing happens only at delivery time. @@ -339,7 +350,7 @@ func wireExtras( } // Wire vault tools and interceptors (conditional on vault store availability) - wireVault(stores, toolsReg, workspace, domainBus) + vaultIntc = wireVault(stores, toolsReg, workspace, domainBus) // Wire delegate tool for inter-agent delegation via agent_links. if stores.AgentLinks != nil && stores.Agents != nil { diff --git a/cmd/gateway_system_config_sync.go b/cmd/gateway_system_config_sync.go index 87068d3c..7e8b5e02 100644 --- a/cmd/gateway_system_config_sync.go +++ b/cmd/gateway_system_config_sync.go @@ -106,6 +106,10 @@ func seedConfigForContext(ctx context.Context, sc store.SystemConfigStore, cfg * setBool("gateway.tool_status", cfg.Gateway.ToolStatus) setInt("gateway.task_recovery_interval_sec", cfg.Gateway.TaskRecoveryIntervalSec) + // Background workers + set("background.provider", cfg.Gateway.BackgroundProvider) + set("background.model", cfg.Gateway.BackgroundModel) + // Tools set("tools.profile", cfg.Tools.Profile) setInt("tools.rate_limit_per_hour", cfg.Tools.RateLimitPerHour) diff --git a/cmd/gateway_vault_wiring.go b/cmd/gateway_vault_wiring.go index 11ff61e0..8b51f486 100644 --- a/cmd/gateway_vault_wiring.go +++ b/cmd/gateway_vault_wiring.go @@ -12,22 +12,17 @@ import ( // wireVault wires Knowledge Vault tools and interceptors into the tool registry. // All wiring is skipped if stores.Vault is nil. // Pattern mirrors wireExtras KG wiring: register tools, set stores, set interceptors. -func wireVault(stores *store.Stores, toolsReg *tools.Registry, workspace string, bus eventbus.DomainEventBus) { +// wireVault wires Knowledge Vault tools and interceptors into the tool registry. +// Returns the shared VaultInterceptor for use by other subsystems (e.g. agent upload hook). +// Returns nil if stores.Vault is nil. +func wireVault(stores *store.Stores, toolsReg *tools.Registry, workspace string, bus eventbus.DomainEventBus) *tools.VaultInterceptor { if stores.Vault == nil { - return + return nil } // Register vault tools — these are always available when vault store is present. vaultSearchTool := tools.NewVaultSearchTool() - vaultLinkTool := tools.NewVaultLinkTool() - vaultBacklinksTool := tools.NewVaultBacklinksTool() toolsReg.Register(vaultSearchTool) - toolsReg.Register(vaultLinkTool) - toolsReg.Register(vaultBacklinksTool) - - // Wire vault store onto link/backlinks tools. - vaultLinkTool.SetVaultStore(stores.Vault) - vaultBacklinksTool.SetVaultStore(stores.Vault) // Build VaultSearchService: fan-out across vault + KG (episodic store pending impl). // EpisodicStore is nil until a PG implementation exists. @@ -78,5 +73,6 @@ func wireVault(stores *store.Stores, toolsReg *tools.Registry, workspace string, } } - slog.Info("vault tools registered", "tools", "vault_search,vault_link,vault_backlinks,create_image,create_video,create_audio,tts,edit") + slog.Info("vault tools registered", "tools", "vault_search,create_image,create_video,create_audio,tts,edit") + return vaultIntc } diff --git a/docs/05-channels-messaging.md b/docs/05-channels-messaging.md index 59aaf28b..d3c8607f 100644 --- a/docs/05-channels-messaging.md +++ b/docs/05-channels-messaging.md @@ -88,9 +88,9 @@ Every channel must implement the base interface: | Interface | Purpose | Implemented By | |-----------|---------|----------------| | `StreamingChannel` | Real-time streaming updates | Telegram, Slack | -| `WebhookChannel` | Webhook HTTP handler mounting | Feishu | +| `WebhookChannel` | Webhook HTTP handler mounting | Facebook, Feishu/Lark, Pancake | | `ReactionChannel` | Status reactions on messages | Telegram, Slack, Feishu | -| `BlockReplyChannel` | Override gateway block_reply setting | Slack | +| `BlockReplyChannel` | Override gateway block_reply setting | Discord, Feishu/Lark, Pancake, Slack, Zalo OA, Zalo Personal | `BaseChannel` provides a shared implementation that all channels embed: allowlist matching, `HandleMessage()`, `CheckPolicy()`, and user ID extraction. diff --git a/docs/06-store-data-model.md b/docs/06-store-data-model.md index 1c69fb4b..e2d03547 100644 --- a/docs/06-store-data-model.md +++ b/docs/06-store-data-model.md @@ -101,6 +101,18 @@ flowchart TD | Cron | `agent:{agentId}:cron:{jobId}:run:{runId}` | `agent:default:cron:reminder:run:abc123` | | Main | `agent:{agentId}:{mainKey}` | `agent:default:main` | +### Session Metadata - Compaction Tracking + +**New well-known metadata key** (Phase 5 follow-up): `last_compaction_at` (RFC3339 string) + +This timestamp is written to `sessions.metadata` JSONB after successful message compaction (context pruning). Both execution paths update it: +- **V3 pipeline**: `PruneStage.CompactMessages()` after successful compaction +- **V2 legacy**: `maybeSummarize()` goroutine after successful summarization + +Operators can read this via `GetSessionMetadata()` to understand when a session was last compacted. The web UI optionally displays this timestamp in a context-usage tooltip. + +Go constant export: `agent.SessionMetaKeyLastCompactionAt = "last_compaction_at"` + --- ## 4. Agent Access Control @@ -661,7 +673,7 @@ L0 (Working Memory) L1 (Episodic Memory) L2 (Semantic Memory) | Table | Purpose | Key Columns | |-------|---------|-------------| -| `episodic_summaries` | Session conversation summaries | `agent_id`, `user_id`, `session_key`, `summary`, `l0_abstract`, `key_topics` (TEXT[]), `embedding` (vector), `source_id` (dedup), `expires_at` | +| `episodic_summaries` | Session conversation summaries | `agent_id`, `user_id`, `session_key`, `summary`, `l0_abstract`, `key_topics` (TEXT[]), `embedding` (vector), `source_id` (dedup), `expires_at`, `recall_count` (INT), `recall_score` (FLOAT), `last_recalled_at` (TIMESTAMPTZ) | | `agent_evolution_metrics` | Self-evolution performance data | `agent_id`, `session_key`, `metric_type` (retrieval/tool/feedback), `metric_key`, `value` (JSONB) | | `agent_evolution_suggestions` | Data-driven improvement suggestions | `agent_id`, `suggestion_type`, `suggestion`, `rationale`, `parameters` (JSONB), `status` (pending/approved/rejected/applied) | | `vault_documents` | Knowledge Vault document registry | `agent_id`, `scope` (personal/team/shared), `path`, `title`, `doc_type`, `content_hash`, `embedding` (vector), `metadata` (JSONB) | @@ -757,7 +769,25 @@ flowchart TD | **EpisodicWorker** | `run.completed` | Extract session summary via LLM or compaction summary. Generate L0 abstract. Store in `episodic_summaries`. Emit `episodic.created` | | **SemanticWorker** | `episodic.created` | Parse summary for entity mentions and relationships. Extract via regex/NER. Insert into KG tables (`kg_entities`, `kg_relations`). Emit `entity.upserted` | | **DedupWorker** | `entity.upserted` | Check for duplicate entities via embedding similarity. Merge duplicate nodes by redirecting relations. Update timestamps to reflect consolidation | -| **DreamingWorker** | `episodic.created` (debounced 10m) | Batch collect unpromoted episodic summaries. Call LLM for synthesis/insight pass. Write results to long-term memory (update KG, write to vault, etc.) | +| **DreamingWorker** | `episodic.created` (debounced 10m) | Batch collect unpromoted episodic summaries scored by usefulness (recall signal). Call LLM for synthesis/insight pass. Write results to long-term memory (update KG, write to vault, etc.) | + +### Dreaming Weighted Scoring (Phase 10, Migration 000045) + +The DreamingWorker prioritizes unpromoted episodic summaries by usefulness via a 4-component running-average score: + +**ComputeRecallScore formula** (14-day half-life): +``` +score = 0.30 * frequency + 0.35 * relevance + 0.20 * recency + 0.15 * freshness +``` + +**Tracking columns** (added to `episodic_summaries`): +- `recall_count INT DEFAULT 0` — Number of times this summary was returned in memory searches +- `recall_score DOUBLE PRECISION DEFAULT 0` — Weighted average score (0 to 1) +- `last_recalled_at TIMESTAMPTZ` — Timestamp of most recent search hit + +**Index for DreamingWorker**: `idx_episodic_recall_unpromoted` on `(agent_id, user_id, recall_score DESC) WHERE promoted_at IS NULL`. Enables efficient `ListUnpromotedScored()` queries to fetch highest-scoring summaries first. + +**Integration with memory_search tool**: After search results are returned to agent, a fire-and-forget task increments `recall_count`, updates `recall_score` via running average, and sets `last_recalled_at`. No blocking — search returns immediately. ### Configuration diff --git a/docs/12-extended-thinking.md b/docs/12-extended-thinking.md index bd2808b9..687d90de 100644 --- a/docs/12-extended-thinking.md +++ b/docs/12-extended-thinking.md @@ -227,7 +227,43 @@ OpenAI-compatible providers handle thinking/reasoning content as metadata. The ` --- -## 6. Observability +## 6. Reasoning Content Stripping (Phase 6 — OpenClaw TS port) + +Some models emit chain-of-thought reasoning tokens even when `effort="off"` is specified. To prevent that raw CoT from reaching end users, GoClaw supports a `StripThinking` flag on `ReasoningDecision`. + +### Models Known to Leak CoT + +Auto-flagged via `modelLeaksReasoning(model)` in `internal/providers/reasoning_resolution.go`: +- **Kimi family**: any model name containing `kimi` (case-insensitive, e.g. `kimi-k2`, `moonshot/kimi-k2-thinking`) +- **DeepSeek-Reasoner**: any model name containing `deepseek-reasoner` + +The allowlist is a simple substring check — extendable as new leaky models appear. + +### Auto-Enable Flow + +1. `ResolveReasoningDecision` runs its normal flow, then a `defer` checks: if `EffectiveEffort == "off" && modelLeaksReasoning(model)`, it sets `decision.StripThinking = true`. +2. The agent loop (`loop_pipeline_callbacks.go`) reads `decision.StripThinking` and propagates it into `chatReq.Options[providers.OptStripThinking] = true`. +3. Each provider's streaming handler reads `OptStripThinking` from options at the top of `ChatStream`/`Chat` and applies guard clauses. + +### Implementation Per Provider + +- **Anthropic** (`anthropic_stream.go`, `anthropic.go`): `thinking_delta` events skip `result.Thinking` accumulation AND the `onChunk` emit when stripping, but `thinkingChars` still increments so `Usage.ThinkingTokens` stays billable. `RawAssistantContent` (content blocks for tool-use passback) is never touched. Non-streaming `Chat()` clears `resp.Thinking` post-parse. +- **OpenAI** (`openai_chat.go`): streaming guards `reasoning`/`reasoning_content` delta accumulation; non-streaming clears `resp.Thinking` post-parse. `Usage.ThinkingTokens` read from the usage chunk independently. +- **Codex** (`codex.go`): `processSSEEvent` takes an extra `stripThinking bool` param; the `reasoning` item case skips summary text appending when set. Usage still extracted from `response.completed`. +- **DashScope**: inherits both guards via `OpenAIProvider` embedding — no separate implementation needed. + +### Invariants Preserved + +| Field | Stripping effect | Rationale | +|---|---|---| +| `ChatResponse.Thinking` | Cleared | User-visible output | +| `Usage.ThinkingTokens` | **Unchanged** | Billing accuracy (Phase 1 depends on it) | +| `RawAssistantContent` | **Unchanged** | Anthropic tool-use replay requires raw thinking blocks | +| `onChunk(StreamChunk{Thinking})` | Not emitted | Streaming UI display | + +--- + +## 7. Observability Each LLM span can now include a `metadata.reasoning` section with: - `source` diff --git a/docs/17-changelog.md b/docs/17-changelog.md index c726b274..53d1e186 100644 --- a/docs/17-changelog.md +++ b/docs/17-changelog.md @@ -32,6 +32,59 @@ All notable changes to GoClaw Gateway are documented here. Format follows [Keep ## [Unreleased] +### Added + +#### Episodic Memory Weighted Scoring — Dreaming Enhancement (2026-04-10, Phase 10) +- **Recall signal tracking**: `episodic_summaries` table gains 3 columns: `recall_count INT`, `recall_score DOUBLE PRECISION`, `last_recalled_at TIMESTAMPTZ` to track usefulness of each memory +- **ComputeRecallScore formula**: 4-component running average (30% frequency + 35% relevance + 20% recency + 15% freshness, 14-day half-life) quantifies memory value +- **DreamingWorker prioritization**: `ListUnpromotedScored()` queries sort by `recall_score DESC` instead of `created_at ASC`, promoting high-signal summaries for synthesis +- **fire-and-forget updates**: `memory_search` tool fire-and-forget tasks increment recall counts asynchronously without blocking search results +- **Index optimization**: New partial index `idx_episodic_recall_unpromoted ON episodic_summaries(agent_id, user_id, recall_score DESC) WHERE promoted_at IS NULL` for efficient DreamingWorker queries +- **Migration 000045**: PG schema v44→45 + SQLite schema v12→13 + +#### Compaction Telemetry — Message Context Tracking (2026-04-10, Phase 5 Follow-up) +- **Session metadata tracking**: `sessions.metadata` JSONB gains well-known key `last_compaction_at` (RFC3339 timestamp) after successful message compaction +- **Dual execution paths**: Both v3 `PruneStage.CompactMessages` and v2 legacy `maybeSummarize` update timestamp on successful compaction +- **Operator visibility**: `GetSessionMetadata()` exposes compaction timestamp; web UI shows in context-usage tooltip +- **Go constant export**: `agent.SessionMetaKeyLastCompactionAt = "last_compaction_at"` + +#### Provider Reasoning Content Stripping (2026-04-10, Phase 6) +- **Auto-strip known leakers**: Models known to leak chain-of-thought at effort="off" (Kimi family, DeepSeek-Reasoner) auto-enable `StripThinking` so user-visible `ChatResponse.Thinking` stays empty +- **Multi-provider support**: Guard clauses in Anthropic streaming, Anthropic non-streaming `Chat()`, OpenAI `ChatStream`/`Chat`, Codex `processSSEEvent`; DashScope inherits via OpenAIProvider embedding +- **Billing-safe invariants**: `Usage.ThinkingTokens` still counted from raw bytes; `RawAssistantContent` untouched so Anthropic tool-use passback continues to work +- **Option plumbing**: New `providers.OptStripThinking` key propagated via `ChatRequest.Options`; `ReasoningDecision.StripThinking` auto-set in `ResolveReasoningDecision` defer +- **Helper**: `modelLeaksReasoning(model string) bool` — extensible allowlist + +#### Dreaming Config Per-Agent (2026-04-10, Phase 8) +- **Per-agent overrides**: `MemoryConfig.Dreaming` JSONB on `agents.memory_config` (nested, no migration) controls dreaming worker behaviour per-agent +- **Fields**: `Enabled *bool`, `DebounceMs int`, `Threshold int`, `VerboseLog *bool` — all pointer/zero-default for partial-override merge semantics +- **Resolver pattern**: `DreamingConfigResolver func(ctx, agentID) *DreamingConfig` wired via `newAgentStoreResolver(AgentStore)`. `ConsolidationDeps` gains optional `AgentStore store.AgentCRUDStore` +- **Backward compat**: Nil resolver → struct defaults; empty JSONB → defaults via zero-value short-circuit +- **Merge helper**: `mergeDreamingConfig` applies override fields only when explicitly set, preserving base values + +#### Per-Provider Context Window & Hardening (2026-04-10, Phase 4, commit 8d37dc45) +- **EffectiveContextWindow**: Resolved once per run in ContextStage from `ModelRegistry` (via provider+model lookup); `PruneStage` uses it with fallback to `Config.ContextWindow` +- **ReserveTokens safety buffer**: New `PipelineConfig.ReserveTokens` subtracted from history budget so PruneStage compacts slightly before the hard limit +- **InMemoryCache hardening**: TTL sweep (60s) + max-size cap (10k) with oldest-first eviction; `Close()` wired to gateway shutdown +- **ContactCollector tenant fix**: Cache key now includes `tenantID + channelInstance` (was silently skipping upserts for same sender across tenants) + +#### Context-Aware Auto-Inject Query (2026-04-10, Phase 9, commit 2731f99a) +- **RecentContext enrichment**: `InjectParams.RecentContext` field supplies last 1-2 user turns; `pgAutoInjector.Inject` prepends "Context:" frame before "Query:" focus +- **Helper**: `pipeline.buildRecentContext()` walks history backward, picks last 2 user turns, caps at 300 chars +- **Callback signature**: `AutoInject func(ctx, msg, userID, recentContext string)` — additive, backward compatible +- **Why**: Vector search on "what's my favorite?" alone returns nothing useful; context-aware query captures conversational intent + +#### Cost Calculation Thinking Tokens Fix (2026-04-10, Phase 1, commit 77a80680) +- **Critical billing bug**: `CalculateCost` now properly handles `ThinkingTokens` as sub-count of `CompletionTokens` (not double-counted for OpenAI o3/o4-mini, Codex/GPT-5; properly split for Anthropic extended thinking) +- **Provider-aware**: Splits only when `ReasoningPerMillion > 0`, otherwise leaves as-is (default matches provider billing) + +#### Web UI Enhancements (2026-04-10, Phase 11) +- **Context usage badge**: Chat top bar shows `{used}/{max} ({percent}%)` with color ramp (amber ≥75%, destructive ≥90%); hidden on mobile via `hidden sm:flex` +- **Compaction indicator**: Context badge tooltip includes compaction count + last compaction timestamp (read from `session.metadata.last_compaction_at`) +- **DreamingConfig UI**: Agent detail MemorySection renders nested dreaming block (4 controls: enabled, threshold, debounce_ms, verbose_log) +- **i18n**: New keys in en/vi/zh: `agents.configSections.dreaming.*`, `chat.contextUsage.*` +- **Not shipped (YAGNI)**: "Memory recall config section" (existing MemorySection already covers all MemoryConfig fields), "Session types extension" (fields already present or out of scope) + ### Refactored #### V3 Architecture Refactor — Phase 6 Completion (2026-04-08) @@ -233,9 +286,11 @@ All notable changes to GoClaw Gateway are documented here. Format follows [Keep - **File viewer**: Improved workspace file view/download and storage depth control - **Pairing DB errors**: Handle transient errors gracefully - **Provider thinking**: Corrected DashScope per-model thinking logic +- **Pancake Page loop guard**: Narrowed webhook ingress to `messaging` + `INBOX` events and normalized HTML-formatted echoes before short-TTL outbound echo suppression, reducing Facebook Page self-reply loops in Pancake inbox conversations ### Documentation +- Updated `05-channels-messaging.md` — Refreshed `WebhookChannel` / `BlockReplyChannel` implementation tables for Facebook, Pancake, Discord, and Zalo-family channels - Updated `18-http-api.md` — Added section 17 for Runtime & Packages Management endpoints - Updated `09-security.md` — Added Docker entrypoint documentation, pkg-helper architecture, privilege separation - Updated `17-changelog.md` — New entries for packages management, Docker security, and auth fix diff --git a/internal/agent/evolution_guardrails.go b/internal/agent/evolution_guardrails.go index a8c84bf5..27c1f296 100644 --- a/internal/agent/evolution_guardrails.go +++ b/internal/agent/evolution_guardrails.go @@ -6,7 +6,6 @@ import ( "fmt" "log/slog" "maps" - "math" "slices" "github.com/google/uuid" @@ -55,22 +54,8 @@ func CheckGuardrails(g AdaptationGuardrails, sg store.EvolutionSuggestion, dataP } } - // Check delta constraint for threshold-type suggestions. - if sg.SuggestionType == store.SuggestThreshold { - maxDelta := g.MaxDeltaPerCycle - if maxDelta <= 0 { - maxDelta = 0.1 - } - var params map[string]any - if err := json.Unmarshal(sg.Parameters, ¶ms); err == nil { - if currentRate, ok := params["current_usage_rate"].(float64); ok { - // Proposed change is bounded by max delta. - if math.Abs(currentRate) > maxDelta { - return fmt.Errorf("delta %.2f exceeds max %.2f per cycle", currentRate, maxDelta) - } - } - } - } + // Threshold suggestions: the applied delta is capped at MaxDeltaPerCycle in ApplySuggestion, + // so no additional delta check is needed here. MinDataPoints and LockedParams already guard safety. return nil } @@ -78,7 +63,7 @@ func CheckGuardrails(g AdaptationGuardrails, sg store.EvolutionSuggestion, dataP // ApplySuggestion applies an approved suggestion's parameters to the agent's other_config JSONB. // Stores the previous values in the suggestion's parameters for rollback. // Scope: only retrieval-related parameters. Never security or core settings. -func ApplySuggestion(ctx context.Context, agentStore store.AgentStore, sugStore store.EvolutionSuggestionStore, sg store.EvolutionSuggestion) error { +func ApplySuggestion(ctx context.Context, agentStore store.AgentStore, sugStore store.EvolutionSuggestionStore, sg store.EvolutionSuggestion, guardrails AdaptationGuardrails) error { // Load current agent config. agent, err := agentStore.GetByID(ctx, sg.AgentID) if err != nil { @@ -102,12 +87,20 @@ func ApplySuggestion(ctx context.Context, agentStore store.AgentStore, sugStore // Apply suggestion-specific changes based on type. switch sg.SuggestionType { case store.SuggestThreshold: - // Raise retrieval threshold slightly (bounded by guardrails). + // Raise retrieval threshold by MaxDeltaPerCycle (bounded by guardrails). current, _ := otherConfig["retrieval_threshold"].(float64) if current == 0 { current = 0.3 // default threshold } - otherConfig["retrieval_threshold"] = current + 0.05 // conservative bump + delta := guardrails.MaxDeltaPerCycle + if delta <= 0 { + delta = 0.05 // fallback + } + newThreshold := current + delta + if newThreshold > 0.95 { + newThreshold = 0.95 + } + otherConfig["retrieval_threshold"] = newThreshold default: // Non-threshold suggestions are informational only, no auto-apply. return fmt.Errorf("suggestion type %q does not support auto-apply", sg.SuggestionType) diff --git a/internal/agent/evolution_guardrails_test.go b/internal/agent/evolution_guardrails_test.go index f7892059..6a5f0018 100644 --- a/internal/agent/evolution_guardrails_test.go +++ b/internal/agent/evolution_guardrails_test.go @@ -2,6 +2,7 @@ package agent import ( "encoding/json" + "strings" "testing" "github.com/nextlevelbuilder/goclaw/internal/store" @@ -51,7 +52,7 @@ func TestCheckGuardrails(t *testing.T) { dataPoints: 200, params: map[string]any{"source": "mem"}, sgType: store.SuggestThreshold, - wantErr: `parameter "source" is locked`, + wantErr: "parameter \"source\" is locked", }, { name: "no locked params passes", @@ -67,6 +68,14 @@ func TestCheckGuardrails(t *testing.T) { sgType: store.SuggestThreshold, wantErr: "insufficient data", }, + { + name: "threshold with usage rate passes (no broken delta check)", + guardrails: DefaultGuardrails(), + dataPoints: 200, + params: map[string]any{"current_usage_rate": 0.15, "source": "vault"}, + sgType: store.SuggestThreshold, + // Previously this would fail with "delta 0.15 exceeds max 0.10" — now passes. + }, } for _, tt := range tests { @@ -84,24 +93,10 @@ func TestCheckGuardrails(t *testing.T) { t.Errorf("expected error containing %q, got nil", tt.wantErr) } if tt.wantErr != "" && err != nil { - if got := err.Error(); !contains(got, tt.wantErr) { - t.Errorf("error %q does not contain %q", got, tt.wantErr) + if !strings.Contains(err.Error(), tt.wantErr) { + t.Errorf("error %q does not contain %q", err.Error(), tt.wantErr) } } }) } } - -func contains(s, substr string) bool { - return len(s) >= len(substr) && (s == substr || len(substr) == 0 || - (len(s) > 0 && len(substr) > 0 && searchStr(s, substr))) -} - -func searchStr(s, sub string) bool { - for i := 0; i <= len(s)-len(sub); i++ { - if s[i:i+len(sub)] == sub { - return true - } - } - return false -} diff --git a/internal/agent/loop_context.go b/internal/agent/loop_context.go index 37b40b84..1bf6f88f 100644 --- a/internal/agent/loop_context.go +++ b/internal/agent/loop_context.go @@ -115,7 +115,8 @@ func (l *Loop) injectContext(ctx context.Context, req *RunRequest) (contextSetup // Seeding must run before buildMessages→resolveContextFiles reads context files. // Team sessions skip seeding: members process tasks from leader, not end-user onboarding. isTeamSession := bootstrap.IsTeamSession(req.SessionKey) - setup := l.getOrCreateUserSetup(ctx, req.UserID, req.Channel, isTeamSession) + channelMeta := l.buildChannelMeta(req) + setup := l.getOrCreateUserSetup(ctx, req.UserID, req.Channel, isTeamSession, channelMeta) // Workspace resolution (layered pipeline). // Layer order: tenant → team → project (future) → user/chat diff --git a/internal/agent/loop_history_sanitize.go b/internal/agent/loop_history_sanitize.go index 4830f7fe..13acea95 100644 --- a/internal/agent/loop_history_sanitize.go +++ b/internal/agent/loop_history_sanitize.go @@ -295,6 +295,11 @@ func (l *Loop) maybeSummarize(ctx context.Context, sessionKey string) { l.sessions.SetSummary(sctx, sessionKey, SanitizeAssistantContent(resp.Content)) l.sessions.TruncateHistory(sctx, sessionKey, keepLast) l.sessions.IncrementCompaction(sctx, sessionKey) + // Mirror SessionMetaKeyLastCompactionAt from the v3 prune/compact path + // so the legacy v2 post-turn summarizer also surfaces compaction cadence. + l.sessions.SetSessionMetadata(sctx, sessionKey, map[string]string{ + SessionMetaKeyLastCompactionAt: time.Now().UTC().Format(time.RFC3339), + }) l.sessions.Save(sctx, sessionKey) }() } diff --git a/internal/agent/loop_input_media.go b/internal/agent/loop_input_media.go index 75b5d56c..3797fc9a 100644 --- a/internal/agent/loop_input_media.go +++ b/internal/agent/loop_input_media.go @@ -3,12 +3,28 @@ package agent import ( "context" "log/slog" + "os" + "strings" "github.com/nextlevelbuilder/goclaw/internal/bus" "github.com/nextlevelbuilder/goclaw/internal/providers" "github.com/nextlevelbuilder/goclaw/internal/tools" ) +// isTextMime returns true for MIME types representing human-readable text content. +// Covers text/* family plus common application/* types that are text-based. +func isTextMime(mime string) bool { + if strings.HasPrefix(mime, "text/") { + return true + } + switch mime { + case "application/json", "application/xml", "application/yaml", + "application/x-yaml", "application/javascript": + return true + } + return false +} + // collectRefsByKind gathers MediaRefs of a given kind from message history // (reverse order) and current-turn refs. Historical first, current last. func collectRefsByKind(messages []providers.Message, currentRefs []providers.MediaRef, kind string) []providers.MediaRef { @@ -47,6 +63,18 @@ func (l *Loop) enrichInputMedia(ctx context.Context, req *RunRequest, messages [ var mediaRefs []providers.MediaRef if len(req.Media) > 0 { mediaRefs = l.persistMedia(req.SessionKey, req.Media, tools.ToolWorkspaceFromCtx(ctx)) + + // Register persisted text uploads in vault (async, non-blocking). + if l.onTextUploaded != nil { + for _, ref := range mediaRefs { + if ref.Path != "" && isTextMime(ref.MimeType) { + if content, err := os.ReadFile(ref.Path); err == nil { + go l.onTextUploaded(context.WithoutCancel(ctx), ref.Path, string(content)) + } + } + } + } + // Load current-turn images from persisted refs (Path is always set for new uploads). var imageFiles []bus.MediaFile for _, ref := range mediaRefs { diff --git a/internal/agent/loop_mcp_user.go b/internal/agent/loop_mcp_user.go index ce6ba393..11a536f1 100644 --- a/internal/agent/loop_mcp_user.go +++ b/internal/agent/loop_mcp_user.go @@ -86,7 +86,7 @@ func (l *Loop) getUserMCPTools(ctx context.Context, userID string) []tools.Tool // shared tool registry so ExecuteWithContext can resolve them by name. reg, _ := l.tools.(*tools.Registry) for _, mcpTool := range entry.MCPTools() { - bt := mcpbridge.NewBridgeTool(srv.Name, mcpTool, entry.Client(), srv.ToolPrefix, srv.TimeoutSec, entry.Connected()) + bt := mcpbridge.NewBridgeTool(srv.Name, mcpTool, entry.ClientPtr(), srv.ToolPrefix, srv.TimeoutSec, entry.Connected()) // Register in registry so ExecuteWithContext can find them. // Skip if already registered (another user loaded this server with same tool names). if reg != nil { diff --git a/internal/agent/loop_pipeline_adapter.go b/internal/agent/loop_pipeline_adapter.go index a1794011..33778445 100644 --- a/internal/agent/loop_pipeline_adapter.go +++ b/internal/agent/loop_pipeline_adapter.go @@ -59,6 +59,19 @@ func (l *Loop) buildPipelineDeps(req *RunRequest, bridgeRS *runState) pipeline.P Compaction: l.compactionCfg, // V3 memory/retrieval flags removed — always true at runtime. }, + // Resolve per-model context window once per run. Falls back to + // Config.ContextWindow when registry/model is unknown (existing + // behaviour unchanged for tests and lite edition). + ResolveContextWindow: func(provider, model string) int { + if l.modelRegistry == nil || model == "" { + return 0 + } + spec := l.modelRegistry.Resolve(provider, model) + if spec == nil { + return 0 + } + return spec.ContextWindow + }, EmitEvent: func(event any) { if ae, ok := event.(AgentEvent); ok { l.emit(ae) @@ -233,16 +246,19 @@ func convertRunResult(pr *pipeline.RunResult) *RunResult { // makeAutoInjectCallback creates the AutoInject callback that captures agent/tenant context. // Returns nil if autoInjector is not configured (v3 retrieval disabled or no episodic store). -func (l *Loop) makeAutoInjectCallback(req *RunRequest) func(ctx context.Context, userMessage, userID string) (string, error) { +// Phase 9: plumbs recentContext through to enrich vector search queries for +// context-aware recall. +func (l *Loop) makeAutoInjectCallback(req *RunRequest) func(ctx context.Context, userMessage, userID, recentContext string) (string, error) { if l.autoInjector == nil { return nil } - return func(ctx context.Context, userMessage, userID string) (string, error) { + return func(ctx context.Context, userMessage, userID, recentContext string) (string, error) { result, err := l.autoInjector.Inject(ctx, memory.InjectParams{ - AgentID: l.agentUUID.String(), - UserID: userID, - TenantID: store.TenantIDFromContext(ctx).String(), - UserMessage: userMessage, + AgentID: l.agentUUID.String(), + UserID: userID, + TenantID: store.TenantIDFromContext(ctx).String(), + UserMessage: userMessage, + RecentContext: recentContext, }) if err != nil || result == nil { return "", err diff --git a/internal/agent/loop_pipeline_callbacks.go b/internal/agent/loop_pipeline_callbacks.go index d3230e9c..abc93157 100644 --- a/internal/agent/loop_pipeline_callbacks.go +++ b/internal/agent/loop_pipeline_callbacks.go @@ -47,7 +47,7 @@ func (l *Loop) pipelineCallbacks(req *RunRequest, bridgeRS *runState) pipelineCa buildFilteredTools: l.makeBuildFilteredTools(req), callLLM: l.makeCallLLM(req, emitRun), pruneMessages: l.makePruneMessages(), - compactMessages: l.makeCompactMessages(), + compactMessages: l.makeCompactMessages(req), runMemoryFlush: l.makeRunMemoryFlush(), executeToolCall: l.makeExecuteToolCall(req, bridgeRS), executeToolRaw: l.makeExecuteToolRaw(req), @@ -241,6 +241,9 @@ func (l *Loop) makeCallLLM(req *RunRequest, emitRun func(AgentEvent)) func(ctx c if effort := reasoningDecision.RequestEffort(); effort != "" { chatReq.Options[providers.OptThinkingLevel] = effort } + if reasoningDecision.StripThinking { + chatReq.Options[providers.OptStripThinking] = true + } // Emit LLM span start for tracing. start := time.Now().UTC() @@ -309,16 +312,30 @@ func (l *Loop) makePruneMessages() func(msgs []providers.Message, budget int) [] } } -func (l *Loop) makeCompactMessages() func(ctx context.Context, msgs []providers.Message, model string) ([]providers.Message, error) { +func (l *Loop) makeCompactMessages(req *RunRequest) func(ctx context.Context, msgs []providers.Message, model string) ([]providers.Message, error) { return func(ctx context.Context, msgs []providers.Message, model string) ([]providers.Message, error) { compacted := l.compactMessagesInPlace(ctx, msgs) if compacted == nil { return msgs, nil // compaction failed, return original } + // Stamp session metadata with the compaction timestamp so operators + // can diagnose compaction cadence without a dedicated column. Stored + // as RFC3339 string in sessions.metadata JSONB (flushed on next save). + if l.sessions != nil && req != nil && req.SessionKey != "" { + l.sessions.SetSessionMetadata(ctx, req.SessionKey, map[string]string{ + SessionMetaKeyLastCompactionAt: time.Now().UTC().Format(time.RFC3339), + }) + } return compacted, nil } } +// SessionMetaKeyLastCompactionAt is the sessions.metadata JSONB key used to +// record the RFC3339 timestamp of the most recent compaction. Exported so +// the web UI code path can read it back via GetSessionMetadata without +// duplicating the string. +const SessionMetaKeyLastCompactionAt = "last_compaction_at" + func (l *Loop) makeRunMemoryFlush() func(ctx context.Context, state *pipeline.RunState) error { return func(ctx context.Context, state *pipeline.RunState) error { settings := ResolveMemoryFlushSettings(l.compactionCfg) diff --git a/internal/agent/loop_types.go b/internal/agent/loop_types.go index 7b9b22d2..acef5aeb 100644 --- a/internal/agent/loop_types.go +++ b/internal/agent/loop_types.go @@ -45,7 +45,8 @@ type EnsureUserProfileFunc func(ctx context.Context, agentID uuid.UUID, userID, // Called once per user per Loop instance, independent of workspace. // isNew indicates whether the profile was just created (seed all) or already existed // (only seed if user has zero files — avoids re-seeding after BOOTSTRAP.md cleanup). -type SeedUserFilesFunc func(ctx context.Context, agentID uuid.UUID, userID, agentType string, isNew bool) error +// channelMeta carries optional channel-provided contact info for bootstrap skip decisions. +type SeedUserFilesFunc func(ctx context.Context, agentID uuid.UUID, userID, agentType string, isNew bool, channelMeta *bootstrap.ChannelMeta) error // EnsureUserFilesFunc is the legacy combined callback (profile + seed + workspace). // Deprecated: use EnsureUserProfileFunc + SeedUserFilesFunc separately. @@ -72,8 +73,10 @@ type Loop struct { agentUUID uuid.UUID // set for context propagation tenantID uuid.UUID // agent's owning tenant agentType string // "open" or "predefined" + defaultTimezone string // system default timezone for bootstrap pre-fill provider providers.Provider model string + modelRegistry providers.ModelRegistry // resolves per-model context window at run time (nil = use static contextWindow) contextWindow int maxTokens int // max output tokens per LLM call (0 = default 8192) maxIterations int @@ -186,6 +189,10 @@ type Loop struct { // Secure CLI store for credentialed exec context injection secureCLIStore store.SecureCLIStore + // Vault hook: called when a text file is persisted from user upload. + // Enables vault registration without agent package importing vault. + onTextUploaded func(ctx context.Context, path, content string) + // Persistent media storage for cross-turn image/document access mediaStore *media.Store @@ -256,6 +263,10 @@ type LoopConfig struct { MemoryCfg *config.MemoryConfig SandboxCfg *sandbox.Config + // ModelRegistry resolves provider/model → ModelSpec for per-run context + // window lookup. Nil = fall back to static LoopConfig.ContextWindow. + ModelRegistry providers.ModelRegistry + Bus bus.EventPublisher DomainBus eventbus.DomainEventBus // V3 domain event bus for consolidation pipeline Sessions store.SessionStore @@ -299,6 +310,7 @@ type LoopConfig struct { ContextFileLoader ContextFileLoaderFunc BootstrapCleanup BootstrapCleanupFunc CacheInvalidate CacheInvalidateFunc // invalidate context file cache after seeding + DefaultTimezone string // system default timezone for bootstrap pre-fill // Tracing collector (nil = no tracing) TraceCollector *tracing.Collector @@ -339,6 +351,9 @@ type LoopConfig struct { // Secure CLI store for credentialed exec context injection SecureCLIStore store.SecureCLIStore + // Vault hook: called asynchronously when a text file is persisted from user upload. + OnTextUploaded func(ctx context.Context, path, content string) + // Persistent media storage for cross-turn image/document access MediaStore *media.Store @@ -409,6 +424,7 @@ func NewLoop(cfg LoopConfig) *Loop { agentType: cfg.AgentType, provider: cfg.Provider, model: cfg.Model, + modelRegistry: cfg.ModelRegistry, contextWindow: cfg.ContextWindow, maxTokens: cfg.MaxTokens, maxIterations: cfg.MaxIterations, @@ -433,6 +449,7 @@ func NewLoop(cfg LoopConfig) *Loop { skillAllowList: cfg.SkillAllowList, hasMemory: cfg.HasMemory, contextFiles: cfg.ContextFiles, + defaultTimezone: cfg.DefaultTimezone, ensureUserProfile: cfg.EnsureUserProfile, seedUserFiles: cfg.SeedUserFiles, ensureUserFiles: cfg.EnsureUserFiles, @@ -461,6 +478,7 @@ func NewLoop(cfg LoopConfig) *Loop { configPermStore: cfg.ConfigPermStore, teamStore: cfg.TeamStore, secureCLIStore: cfg.SecureCLIStore, + onTextUploaded: cfg.OnTextUploaded, mediaStore: cfg.MediaStore, modelPricing: cfg.ModelPricing, budgetMonthlyCents: cfg.BudgetMonthlyCents, diff --git a/internal/agent/loop_utils.go b/internal/agent/loop_utils.go index 19c6233f..5f229c38 100644 --- a/internal/agent/loop_utils.go +++ b/internal/agent/loop_utils.go @@ -64,10 +64,23 @@ func (l *Loop) shouldShareKnowledgeGraph() bool { return l.workspaceSharing != nil && l.workspaceSharing.ShareKnowledgeGraph } +// buildChannelMeta extracts channel metadata from RunRequest for bootstrap decisions. +// Returns nil when channel type is unknown (preserves normal bootstrap flow). +func (l *Loop) buildChannelMeta(req *RunRequest) *bootstrap.ChannelMeta { + if req == nil || req.ChannelType == "" { + return nil + } + return &bootstrap.ChannelMeta{ + ChannelType: req.ChannelType, + DisplayName: req.SenderName, + DefaultTimezone: l.defaultTimezone, + } +} + // getOrCreateUserSetup returns the cached userSetup for a user, creating it on first call. // On first call: seeds context files (non-team) and resolves workspace from user profile. // On subsequent calls: returns cached setup immediately (no DB calls). -func (l *Loop) getOrCreateUserSetup(ctx context.Context, userID, channel string, isTeamSession bool) *userSetup { +func (l *Loop) getOrCreateUserSetup(ctx context.Context, userID, channel string, isTeamSession bool, channelMeta *bootstrap.ChannelMeta) *userSetup { if userID == "" { return &userSetup{workspace: l.workspace} } @@ -92,7 +105,7 @@ func (l *Loop) getOrCreateUserSetup(ctx context.Context, userID, channel string, } // Step 2: Seed context files (must run before buildMessages reads them). // Passes isNew so SeedUserFiles knows whether to skip existing files. - if err := l.seedUserFiles(ctx, l.agentUUID, userID, l.agentType, isNew); err != nil { + if err := l.seedUserFiles(ctx, l.agentUUID, userID, l.agentType, isNew, channelMeta); err != nil { slog.Warn("failed to seed user context files", "error", err) // Seeding failed (e.g. SQLITE_BUSY after retries). Inject // embedded bootstrap templates in-memory so the first turn diff --git a/internal/agent/preview_prompt.go b/internal/agent/preview_prompt.go index 35a4bdfe..abc71ece 100644 --- a/internal/agent/preview_prompt.go +++ b/internal/agent/preview_prompt.go @@ -4,34 +4,49 @@ import ( "context" "path/filepath" "slices" + "strings" "github.com/google/uuid" "github.com/nextlevelbuilder/goclaw/internal/bootstrap" "github.com/nextlevelbuilder/goclaw/internal/providers" "github.com/nextlevelbuilder/goclaw/internal/store" + "github.com/nextlevelbuilder/goclaw/internal/tokencount" + "github.com/nextlevelbuilder/goclaw/internal/tools" ) // PreviewDeps holds optional dependencies for building a preview system prompt. // All fields are nil-safe — missing deps simply skip resolution for that section. type PreviewDeps struct { - AgentStore store.AgentStore - TeamStore store.TeamStore - AgentLinks store.AgentLinkStore - ProviderReg *providers.Registry - ToolLister interface{ List() []string } - SkillsLoader interface { + AgentStore store.AgentStore + TeamStore store.TeamStore + AgentLinks store.AgentLinkStore + ProviderReg *providers.Registry + SkillAccessStore store.SkillAccessStore + ToolLister interface { + List() []string + Get(name string) (tools.Tool, bool) + Aliases() map[string]string + } + SkillsLoader interface { BuildPinnedSummary(ctx context.Context, names []string) string + BuildSummary(ctx context.Context, allowList []string) string } DataDir string // for team workspace path construction } +// PreviewResult holds the output of BuildPreviewPrompt. +type PreviewResult struct { + Prompt string + ToolDefs []providers.ToolDefinition // tool definitions (schemas) as sent to the LLM +} + // BuildPreviewPrompt builds a system prompt for preview purposes. // Reuses the same BuildSystemPrompt() as the LLM pipeline, resolving as many // fields as possible from agent data + DB stores. Runtime-only fields // (channel, peer kind, session context, credentials) are left at zero values — // BuildSystemPrompt already nil-checks every field. -func BuildPreviewPrompt(ctx context.Context, ag *store.AgentData, mode PromptMode, userID string, deps PreviewDeps) string { +func BuildPreviewPrompt(ctx context.Context, ag *store.AgentData, mode PromptMode, userID string, deps PreviewDeps) PreviewResult { // --- Context files --- var contextFiles []bootstrap.ContextFile if deps.AgentStore != nil { @@ -62,6 +77,63 @@ func BuildPreviewPrompt(ctx context.Context, ag *store.AgentData, mode PromptMod toolNames = fallbackPreviewToolNames } + // --- skill_manage gating (matches loop_history.go:124-131) --- + if !ag.ParseSkillEvolve() { + filtered := make([]string, 0, len(toolNames)) + for _, n := range toolNames { + if n != "skill_manage" { + filtered = append(filtered, n) + } + } + toolNames = filtered + } + + // --- Basic agent tool policy: deny-only (Allow/AlsoAllow/ByProvider not applied). + // Full PolicyEngine requires runtime state (provider name, channel) not available in preview. --- + if toolPolicy := ag.ParseToolsConfig(); toolPolicy != nil && len(toolPolicy.Deny) > 0 { + denySet := make(map[string]bool, len(toolPolicy.Deny)) + for _, d := range toolPolicy.Deny { + denySet[d] = true + } + filtered := make([]string, 0, len(toolNames)) + for _, n := range toolNames { + if !denySet[n] { + filtered = append(filtered, n) + } + } + toolNames = filtered + } + + // --- Alias exclusion (matches loop_history.go:136-146) --- + if deps.ToolLister != nil { + if aliasSet := deps.ToolLister.Aliases(); len(aliasSet) > 0 { + filtered := make([]string, 0, len(toolNames)) + for _, n := range toolNames { + if _, isAlias := aliasSet[n]; !isAlias { + filtered = append(filtered, n) + } + } + toolNames = filtered + } + } + + // --- MCP tool descriptions (matches loop_history_supplement.go:44-58) --- + var mcpToolDescs map[string]string + if deps.ToolLister != nil { + descs := make(map[string]string) + for _, name := range toolNames { + if !strings.HasPrefix(name, "mcp_") || name == "mcp_tool_search" { + continue + } + if tool, ok := deps.ToolLister.Get(name); ok { + descs[name] = tool.Description() + } + } + if len(descs) > 0 { + mcpToolDescs = descs + } + } + // --- Sandbox --- sandboxCfg := ag.ParseSandboxConfig() sandboxEnabled := sandboxCfg != nil && sandboxCfg.Mode != "" && sandboxCfg.Mode != "off" @@ -76,6 +148,32 @@ func BuildPreviewPrompt(ctx context.Context, ag *store.AgentData, mode PromptMod pinnedSummary = deps.SkillsLoader.BuildPinnedSummary(ctx, pinnedSkills) } + // --- Skills summary (BuildSummary + token count) --- + var skillsSummary string + if deps.SkillsLoader != nil { + var skillAllowList []string + if deps.SkillAccessStore != nil { + if accessible, err := deps.SkillAccessStore.ListAccessible(ctx, ag.ID, userID); err == nil { + skillAllowList = make([]string, 0, len(accessible)) + for _, sk := range accessible { + skillAllowList = append(skillAllowList, sk.Slug) + } + } else { + // On error: empty list (no skills). Preview is diagnostic; safer than showing all. + skillAllowList = []string{} + } + } + + summary := deps.SkillsLoader.BuildSummary(ctx, skillAllowList) + if summary != "" { + tokens := tokencount.NewFallbackCounter().Count("claude-3", summary) + if tokens <= skillInlineMaxTokens { + skillsSummary = summary + } + // Over threshold → search-only mode (skillsSummary stays empty) + } + } + // --- Provider contribution --- var providerContrib *providers.PromptContribution if deps.ProviderReg != nil && ag.Provider != "" { @@ -121,8 +219,31 @@ func BuildPreviewPrompt(ctx context.Context, ag *store.AgentData, mode PromptMod } } + // --- Tool definitions (schemas sent to LLM alongside the system prompt) --- + var toolDefs []providers.ToolDefinition + if deps.ToolLister != nil { + for _, name := range toolNames { + if tool, ok := deps.ToolLister.Get(name); ok { + toolDefs = append(toolDefs, tools.ToProviderDef(tool)) + } + } + // Include alias definitions (LLM receives both canonical + aliases) + for alias, canonical := range deps.ToolLister.Aliases() { + if tool, ok := deps.ToolLister.Get(canonical); ok { + toolDefs = append(toolDefs, providers.ToolDefinition{ + Type: "function", + Function: providers.ToolFunctionSchema{ + Name: alias, + Description: tool.Description(), + Parameters: tool.Parameters(), + }, + }) + } + } + } + // --- Build system prompt (same function as LLM pipeline) --- - return BuildSystemPrompt(SystemPromptConfig{ + prompt := BuildSystemPrompt(SystemPromptConfig{ AgentID: ag.AgentKey, AgentUUID: ag.ID.String(), DisplayName: ag.DisplayName, @@ -146,6 +267,8 @@ func BuildPreviewPrompt(ctx context.Context, ag *store.AgentData, mode PromptMod SandboxEnabled: sandboxEnabled, SandboxContainerDir: sandboxContainerDir, PinnedSkillsSummary: pinnedSummary, + SkillsSummary: skillsSummary, + MCPToolDescs: mcpToolDescs, IsTeamContext: isTeamCtx, TeamWorkspace: teamWorkspace, TeamMembers: teamMembers, @@ -154,8 +277,9 @@ func BuildPreviewPrompt(ctx context.Context, ag *store.AgentData, mode PromptMod OrchMode: orchMode, // Runtime-only fields left at zero: Channel, ChannelType, ChatTitle, // PeerKind, OwnerIDs, ExtraPrompt, CredentialCLIContext, IsBootstrap, - // MCPToolDescs, SkillsSummary, SandboxWorkspaceAccess + // SandboxWorkspaceAccess }) + return PreviewResult{Prompt: prompt, ToolDefs: toolDefs} } // mergePreviewUserFiles overlays per-user files onto base agent-level files. diff --git a/internal/agent/preview_prompt_test.go b/internal/agent/preview_prompt_test.go new file mode 100644 index 00000000..70177258 --- /dev/null +++ b/internal/agent/preview_prompt_test.go @@ -0,0 +1,238 @@ +package agent + +import ( + "context" + "errors" + "strings" + "testing" + + "github.com/google/uuid" + + "github.com/nextlevelbuilder/goclaw/internal/store" +) + +func baseAgent() *store.AgentData { + return &store.AgentData{ + BaseModel: store.BaseModel{ID: uuid.New()}, + AgentKey: "test-agent", + AgentType: store.AgentTypePredefined, + Workspace: "/workspace", + } +} + +func TestBuildPreviewPrompt_NilDeps(t *testing.T) { + r := BuildPreviewPrompt(context.Background(), baseAgent(), PromptFull, "", PreviewDeps{}) + if r.Prompt == "" { + t.Fatal("expected non-empty prompt with nil deps") + } + if !strings.Contains(r.Prompt, "read_file") { + t.Error("expected fallback tool names in prompt") + } + // No tool lister → no tool defs + if len(r.ToolDefs) != 0 { + t.Errorf("expected no tool defs with nil ToolLister, got %d", len(r.ToolDefs)) + } +} + +func TestBuildPreviewPrompt_SkillsInline(t *testing.T) { + r := BuildPreviewPrompt(context.Background(), baseAgent(), PromptFull, "", PreviewDeps{ + SkillsLoader: &mockSkillsLoader{ + summary: "\nGit operations\n", + }, + }) + if !strings.Contains(r.Prompt, "") { + t.Error("expected skills XML inlined in prompt") + } +} + +func TestBuildPreviewPrompt_SkillsSearchMode(t *testing.T) { + bigSummary := strings.Repeat("x", 10000) + r := BuildPreviewPrompt(context.Background(), baseAgent(), PromptFull, "", PreviewDeps{ + SkillsLoader: &mockSkillsLoader{summary: bigSummary}, + }) + if strings.Contains(r.Prompt, bigSummary) { + t.Error("expected large summary to be excluded (search-only mode)") + } +} + +func TestBuildPreviewPrompt_PinnedSkillsHybrid(t *testing.T) { + ag := baseAgent() + ag.OtherConfig = []byte(`{"pinned_skills":["deploy"]}`) + r := BuildPreviewPrompt(context.Background(), ag, PromptFull, "", PreviewDeps{ + SkillsLoader: &mockSkillsLoader{ + pinned: "Deploy to prod", + summary: "\nGit ops\n", + }, + }) + if !strings.Contains(r.Prompt, "deploy") || !strings.Contains(r.Prompt, "Pinned skills") { + t.Error("expected pinned skills section in prompt") + } +} + +func TestBuildPreviewPrompt_SkillAllowList(t *testing.T) { + ag := baseAgent() + loader := &mockSkillsLoader{ + summary: "ok", + } + r := BuildPreviewPrompt(context.Background(), ag, PromptFull, "user1", PreviewDeps{ + SkillsLoader: loader, + SkillAccessStore: &mockSkillAccessStore{ + accessible: []store.SkillInfo{{Slug: "allowed-skill"}}, + }, + }) + if !strings.Contains(r.Prompt, "") { + t.Error("expected filtered skills in prompt") + } + if len(loader.capturedAllow) != 1 || loader.capturedAllow[0] != "allowed-skill" { + t.Errorf("expected allow list [allowed-skill], got %v", loader.capturedAllow) + } +} + +func TestBuildPreviewPrompt_SkillAccessStoreError(t *testing.T) { + ag := baseAgent() + loader := &mockSkillsLoader{ + summary: "desc", + } + r := BuildPreviewPrompt(context.Background(), ag, PromptFull, "user1", PreviewDeps{ + SkillsLoader: loader, + SkillAccessStore: &mockSkillAccessStore{err: errors.New("db error")}, + }) + if r.Prompt == "" { + t.Fatal("expected non-empty prompt on SkillAccessStore error") + } + if loader.capturedAllow == nil || len(loader.capturedAllow) != 0 { + t.Errorf("expected empty (non-nil) allow list on error, got %v", loader.capturedAllow) + } +} + +func TestBuildPreviewPrompt_MCPToolDescs(t *testing.T) { + r := BuildPreviewPrompt(context.Background(), baseAgent(), PromptFull, "", PreviewDeps{ + ToolLister: &mockToolLister{ + tools: map[string]string{ + "read_file": "Read a file", + "mcp_pg_query": "Run PostgreSQL queries", + }, + }, + }) + if !strings.Contains(r.Prompt, "mcp_pg_query") { + t.Error("expected MCP tool description in prompt") + } +} + +func TestBuildPreviewPrompt_MCPToolSearchExcluded(t *testing.T) { + r := BuildPreviewPrompt(context.Background(), baseAgent(), PromptFull, "", PreviewDeps{ + ToolLister: &mockToolLister{ + tools: map[string]string{ + "read_file": "Read a file", + "mcp_tool_search": "Search MCP tools", + }, + }, + }) + if strings.Contains(r.Prompt, "Search MCP tools") { + t.Error("mcp_tool_search should not appear in MCP tool descriptions") + } +} + +func TestBuildPreviewPrompt_AliasExclusion(t *testing.T) { + r := BuildPreviewPrompt(context.Background(), baseAgent(), PromptFull, "", PreviewDeps{ + ToolLister: &mockToolLister{ + tools: map[string]string{ + "read_file": "Read a file", + "Read": "Alias for read_file", + "exec": "Execute shell", + "Bash": "Alias for exec", + }, + aliases: map[string]string{ + "Read": "read_file", + "Bash": "exec", + }, + }, + }) + if strings.Contains(r.Prompt, "- Read\n") || strings.Contains(r.Prompt, "- Bash\n") { + t.Error("aliases should be excluded from tool list") + } +} + +func TestBuildPreviewPrompt_SkillManageGating(t *testing.T) { + ag := baseAgent() + r := BuildPreviewPrompt(context.Background(), ag, PromptFull, "", PreviewDeps{ + ToolLister: &mockToolLister{ + tools: map[string]string{ + "read_file": "Read a file", + "skill_manage": "Manage skills", + }, + }, + }) + if strings.Contains(r.Prompt, "skill_manage") { + t.Error("skill_manage should be excluded when skill_evolve is off") + } +} + +func TestBuildPreviewPrompt_SkillManageEnabled(t *testing.T) { + ag := baseAgent() + ag.SkillEvolve = true + r := BuildPreviewPrompt(context.Background(), ag, PromptFull, "", PreviewDeps{ + ToolLister: &mockToolLister{ + tools: map[string]string{ + "read_file": "Read a file", + "skill_manage": "Manage skills", + "skill_search": "Search skills", + }, + }, + }) + if !strings.Contains(r.Prompt, "skill_manage") { + t.Error("skill_manage should be present when skill_evolve is on") + } +} + +func TestBuildPreviewPrompt_ToolPolicyDeny(t *testing.T) { + ag := baseAgent() + ag.ToolsConfig = []byte(`{"deny":["exec","web_fetch"]}`) + r := BuildPreviewPrompt(context.Background(), ag, PromptFull, "", PreviewDeps{ + ToolLister: &mockToolLister{ + tools: map[string]string{ + "read_file": "Read a file", + "exec": "Execute shell", + "web_fetch": "Fetch web page", + }, + }, + }) + if strings.Contains(r.Prompt, "- exec\n") { + t.Error("denied tool 'exec' should be excluded") + } + if strings.Contains(r.Prompt, "- web_fetch\n") { + t.Error("denied tool 'web_fetch' should be excluded") + } + if !strings.Contains(r.Prompt, "read_file") { + t.Error("non-denied tool 'read_file' should be present") + } +} + +func TestBuildPreviewPrompt_ToolDefs(t *testing.T) { + r := BuildPreviewPrompt(context.Background(), baseAgent(), PromptFull, "", PreviewDeps{ + ToolLister: &mockToolLister{ + tools: map[string]string{ + "read_file": "Read a file", + "exec": "Execute shell", + }, + aliases: map[string]string{ + "Read": "read_file", + }, + }, + }) + // Should have canonical tools + aliases in tool defs + if len(r.ToolDefs) != 3 { // read_file + exec + Read alias + t.Errorf("expected 3 tool defs (2 canonical + 1 alias), got %d", len(r.ToolDefs)) + } + // Verify alias is included in defs even though excluded from system prompt + found := false + for _, td := range r.ToolDefs { + if td.Function.Name == "Read" { + found = true + break + } + } + if !found { + t.Error("expected alias 'Read' in tool defs") + } +} diff --git a/internal/agent/preview_prompt_test_helpers_test.go b/internal/agent/preview_prompt_test_helpers_test.go new file mode 100644 index 00000000..e5705a32 --- /dev/null +++ b/internal/agent/preview_prompt_test_helpers_test.go @@ -0,0 +1,78 @@ +package agent + +import ( + "context" + "sort" + + "github.com/google/uuid" + + "github.com/nextlevelbuilder/goclaw/internal/store" + "github.com/nextlevelbuilder/goclaw/internal/tools" +) + +// mockToolLister implements the widened ToolLister interface for testing. +type mockToolLister struct { + tools map[string]string // name → description + aliases map[string]string // alias → canonical +} + +func (m *mockToolLister) List() []string { + names := make([]string, 0, len(m.tools)) + for n := range m.tools { + names = append(names, n) + } + sort.Strings(names) + return names +} + +func (m *mockToolLister) Get(name string) (tools.Tool, bool) { + desc, ok := m.tools[name] + if !ok { + return nil, false + } + return &mockTool{name: name, desc: desc}, true +} + +func (m *mockToolLister) Aliases() map[string]string { + if m.aliases == nil { + return nil + } + return m.aliases +} + +// mockTool is a minimal tools.Tool implementation for testing. +type mockTool struct { + name string + desc string +} + +func (t *mockTool) Name() string { return t.name } +func (t *mockTool) Description() string { return t.desc } +func (t *mockTool) Parameters() map[string]any { return nil } +func (t *mockTool) Execute(_ context.Context, _ map[string]any) *tools.Result { return nil } + +// mockSkillsLoader implements the widened SkillsLoader interface. +type mockSkillsLoader struct { + pinned string // pre-built pinned XML + summary string // pre-built full summary + capturedAllow []string // set by BuildSummary for test assertions +} + +func (m *mockSkillsLoader) BuildPinnedSummary(_ context.Context, _ []string) string { + return m.pinned +} + +func (m *mockSkillsLoader) BuildSummary(_ context.Context, allowList []string) string { + m.capturedAllow = allowList + return m.summary +} + +// mockSkillAccessStore returns canned skill access lists. +type mockSkillAccessStore struct { + accessible []store.SkillInfo + err error +} + +func (m *mockSkillAccessStore) ListAccessible(_ context.Context, _ uuid.UUID, _ string) ([]store.SkillInfo, error) { + return m.accessible, m.err +} diff --git a/internal/agent/resolver.go b/internal/agent/resolver.go index 9a7ed1aa..fbe49d57 100644 --- a/internal/agent/resolver.go +++ b/internal/agent/resolver.go @@ -30,6 +30,7 @@ type ResolverDeps struct { AgentStore store.AgentStore ProviderStore store.ProviderStore ProviderReg *providers.Registry + ModelRegistry providers.ModelRegistry // per-model context window + capabilities lookup Bus bus.EventPublisher Sessions store.SessionStore Tools *tools.Registry @@ -45,6 +46,7 @@ type ResolverDeps struct { ContextFileLoader ContextFileLoaderFunc BootstrapCleanup BootstrapCleanupFunc CacheInvalidate CacheInvalidateFunc + DefaultTimezone string // system default timezone for bootstrap pre-fill // Security InjectionAction string // "log", "warn", "block", "off" @@ -115,6 +117,9 @@ type ResolverDeps struct { // V3 domain event bus for consolidation pipeline (nil = disabled) DomainBus eventbus.DomainEventBus + + // Vault hook: called when a text file is uploaded by user (nil = no vault registration) + OnTextUploaded func(ctx context.Context, path, content string) } // NewManagedResolver creates a ResolverFunc that builds Loops from DB agent data. @@ -411,6 +416,7 @@ func NewManagedResolver(deps ResolverDeps) ResolverFunc { AutoInjector: deps.AutoInjector, Provider: provider, Model: ag.Model, + ModelRegistry: deps.ModelRegistry, ContextWindow: contextWindow, MaxTokens: ag.ParseMaxTokens(), MaxIterations: maxIter, @@ -435,6 +441,7 @@ func NewManagedResolver(deps ResolverDeps) ResolverFunc { ContextFileLoader: deps.ContextFileLoader, BootstrapCleanup: deps.BootstrapCleanup, CacheInvalidate: deps.CacheInvalidate, + DefaultTimezone: deps.DefaultTimezone, OnEvent: deps.OnEvent, TraceCollector: deps.TraceCollector, InjectionAction: deps.InjectionAction, @@ -457,6 +464,7 @@ func NewManagedResolver(deps ResolverDeps) ResolverFunc { ConfigPermStore: deps.ConfigPermStore, TeamStore: deps.TeamStore, SecureCLIStore: deps.SecureCLIStore, + OnTextUploaded: deps.OnTextUploaded, MediaStore: deps.MediaStore, ModelPricing: deps.ModelPricing, BudgetMonthlyCents: derefInt(ag.BudgetMonthlyCents), diff --git a/internal/agent/suggestion_engine.go b/internal/agent/suggestion_engine.go index d7f9474f..d5d94ed5 100644 --- a/internal/agent/suggestion_engine.go +++ b/internal/agent/suggestion_engine.go @@ -45,6 +45,30 @@ func NewSuggestionEngine(metrics store.EvolutionMetricsStore, suggestions store. } } +// dedupKey uniquely identifies a suggestion by type + metric key (e.g., tool name or source). +type dedupKey struct { + Type store.SuggestionType + Key string +} + +// extractMetricKey returns the distinguishing key from suggestion parameters. +func extractMetricKey(params json.RawMessage, st store.SuggestionType) string { + var p map[string]any + if json.Unmarshal(params, &p) != nil { + return "" + } + switch st { + case store.SuggestThreshold: + s, _ := p["source"].(string) + return s + case store.SuggestToolOrder, store.SuggestSkillAdd: + s, _ := p["tool"].(string) + return s + default: + return "" + } +} + // Analyze runs all rules against a single agent's metrics (7-day window). // Returns newly created suggestions. Skips rules that produce duplicates. func (e *SuggestionEngine) Analyze(ctx context.Context, agentID uuid.UUID) ([]store.EvolutionSuggestion, error) { @@ -65,11 +89,12 @@ func (e *SuggestionEngine) Analyze(ctx context.Context, agentID uuid.UUID) ([]st Since: since, } - // Load existing pending suggestions to avoid duplicates. + // Load existing pending suggestions to avoid duplicates (composite key: type + metric key). existing, _ := e.suggestions.ListSuggestions(ctx, agentID, "pending", 100) - existingTypes := make(map[store.SuggestionType]bool, len(existing)) + existingKeys := make(map[dedupKey]bool, len(existing)) for _, sg := range existing { - existingTypes[sg.SuggestionType] = true + mk := extractMetricKey(sg.Parameters, sg.SuggestionType) + existingKeys[dedupKey{sg.SuggestionType, mk}] = true } var created []store.EvolutionSuggestion @@ -82,8 +107,9 @@ func (e *SuggestionEngine) Analyze(ctx context.Context, agentID uuid.UUID) ([]st if sg == nil { continue } - // Skip if pending suggestion of same type already exists. - if existingTypes[sg.SuggestionType] { + // Skip if pending suggestion with same type + metric key already exists. + mk := extractMetricKey(sg.Parameters, sg.SuggestionType) + if existingKeys[dedupKey{sg.SuggestionType, mk}] { continue } @@ -95,7 +121,7 @@ func (e *SuggestionEngine) Analyze(ctx context.Context, agentID uuid.UUID) ([]st continue } created = append(created, *sg) - existingTypes[sg.SuggestionType] = true + existingKeys[dedupKey{sg.SuggestionType, mk}] = true } return created, nil diff --git a/internal/agent/systemprompt.go b/internal/agent/systemprompt.go index 14386934..ed77172c 100644 --- a/internal/agent/systemprompt.go +++ b/internal/agent/systemprompt.go @@ -205,6 +205,9 @@ var coreToolSummaries = map[string]string{ "team_tasks": "Team task board — track progress, manage dependencies (spawn auto-creates delegation tasks)", "list_group_members": "List all members of the current group chat (Feishu/Lark only)", "create_forum_topic": "Create a forum topic in a Telegram supergroup", + "delegate": "Delegate a task to a linked agent (requires agent_links). See ## Delegation Targets for available agents", + "memory_expand": "Retrieve full session details from episodic memory results — use after memory_search returns episodic hits", + "vault_search": "Search documents in the knowledge vault (hybrid keyword + semantic)", // Tool aliases (edit_file, sessions_spawn, Read, Write, Edit, Bash, etc.) // are registered in the tool registry but excluded from the system prompt diff --git a/internal/bootstrap/seed_store.go b/internal/bootstrap/seed_store.go index 8e3f737a..619c74e7 100644 --- a/internal/bootstrap/seed_store.go +++ b/internal/bootstrap/seed_store.go @@ -2,6 +2,7 @@ package bootstrap import ( "context" + "fmt" "log/slog" "path/filepath" "strings" @@ -9,9 +10,47 @@ import ( "github.com/google/uuid" + "github.com/nextlevelbuilder/goclaw/internal/channels" "github.com/nextlevelbuilder/goclaw/internal/store" ) +// ChannelMeta carries channel-provided contact info for bootstrap decisions. +type ChannelMeta struct { + ChannelType string + DisplayName string + DefaultTimezone string +} + +// shouldSkipBootstrap returns true when channel provides enough user info +// to pre-fill USER.md, making the interactive bootstrap unnecessary. +// Currently only Pancake channel qualifies (provides Facebook profile name). +func shouldSkipBootstrap(meta *ChannelMeta) bool { + return meta != nil && + meta.ChannelType == "pancake" && + meta.DisplayName != "" +} + +// buildPrefilledUser generates USER.md content pre-filled with channel-provided contact info. +func buildPrefilledUser(meta *ChannelMeta) string { + tz := meta.DefaultTimezone + if tz == "" { + tz = "(unknown)" + } + name := channels.SanitizeDisplayName(meta.DisplayName) + return fmt.Sprintf(`# USER.md - About This User + +- **Name:** %s +- **What to call them:** %s +- **Timezone:** %s + +## Context + +_(First contact via %s channel. Profile info auto-filled from channel data.)_ +`, name, name, tz, meta.ChannelType) +} + + + // retryOnBusy retries fn up to 3 times on SQLITE_BUSY errors with 500ms delay. func retryOnBusy(fn func() error) error { var lastErr error @@ -140,7 +179,7 @@ var userSeedFilesPredefined = []string{ // This ensures wizard-configured owner profiles are preserved on first chat. // // Returns the list of file names that were seeded. -func SeedUserFiles(ctx context.Context, agentStore store.AgentStore, agentID uuid.UUID, userID, agentType string, skipIfAnyExist bool) ([]string, error) { +func SeedUserFiles(ctx context.Context, agentStore store.AgentStore, agentID uuid.UUID, userID, agentType string, skipIfAnyExist bool, channelMeta *ChannelMeta) ([]string, error) { files := userSeedFilesOpen if agentType == store.AgentTypePredefined { files = userSeedFilesPredefined @@ -160,6 +199,44 @@ func SeedUserFiles(ctx context.Context, agentStore store.AgentStore, agentID uui return nil, nil } + // Channel-provided contact info: skip bootstrap, pre-fill USER.md directly. + // Currently only Pancake channel (Facebook Messenger) provides enough user info + // (display_name from Facebook profile) to skip the interactive onboarding flow. + if shouldSkipBootstrap(channelMeta) { + userContent := buildPrefilledUser(channelMeta) + if err := retryOnBusy(func() error { + return agentStore.SetUserContextFile(ctx, agentID, userID, UserFile, userContent) + }); err != nil { + return nil, err + } + seeded := []string{UserFile} + + // For open agents: still seed SOUL.md, IDENTITY.md, AGENTS.md (default templates) + // so the agent has its personality/identity files — just skip BOOTSTRAP.md. + if agentType == store.AgentTypeOpen { + for _, name := range userSeedFilesOpen { + if name == BootstrapFile || name == UserFile { + continue // skip bootstrap (the whole point) and user (already filled) + } + content, err := templateFS.ReadFile(filepath.Join("templates", name)) + if err != nil { + continue + } + if err := retryOnBusy(func() error { + return agentStore.SetUserContextFile(ctx, agentID, userID, name, string(content)) + }); err != nil { + return seeded, err + } + seeded = append(seeded, name) + } + } + + slog.Info("bootstrap skipped (channel contact info)", + "agent", agentID, "user", userID, "channel", channelMeta.ChannelType, + "display_name", channelMeta.DisplayName, "files", seeded) + return seeded, nil + } + hasFile := make(map[string]bool, len(existing)) for _, f := range existing { if f.Content != "" { diff --git a/internal/bootstrap/seed_store_test.go b/internal/bootstrap/seed_store_test.go index be179c9d..925be560 100644 --- a/internal/bootstrap/seed_store_test.go +++ b/internal/bootstrap/seed_store_test.go @@ -3,6 +3,7 @@ package bootstrap import ( "context" "maps" + "strings" "testing" "github.com/google/uuid" @@ -116,6 +117,23 @@ func (s *seedStubStore) PropagateContextFile(_ context.Context, _ uuid.UUID, _ s } // ---- Tests ---- +// TestBuildPrefilledUser_SanitizesMarkdownInjection verifies that DisplayName with +// newlines or markdown syntax does not inject into USER.md structure. +func TestBuildPrefilledUser_SanitizesMarkdownInjection(t *testing.T) { + meta := &ChannelMeta{ + ChannelType: "pancake", + DisplayName: "Evil\n- **Admin:** true\n## Override", + DefaultTimezone: "Asia/Ho_Chi_Minh", + } + content := buildPrefilledUser(meta) + if strings.Contains(content, "## Override") { + t.Error("DisplayName markdown injection not sanitized") + } + if strings.Contains(content, "\n- **Admin:**") { + t.Error("DisplayName newline injection not sanitized") + } +} + // TestSeedUserFiles_PredefinedAgent_UsesAgentLevelUserMD is the primary regression test. // When a predefined agent has wizard-written USER.md in agent_context_files, SeedUserFiles // must seed that content into user_context_files — NOT the blank embedded template. @@ -127,7 +145,7 @@ func TestSeedUserFiles_PredefinedAgent_UsesAgentLevelUserMD(t *testing.T) { // Simulate wizard writing USER.md at agent level via agents.files.set as.agentFiles[UserFile] = wizardContent - seeded, err := SeedUserFiles(context.Background(), as, agentID, "user-alice", store.AgentTypePredefined, false) + seeded, err := SeedUserFiles(context.Background(), as, agentID, "user-alice", store.AgentTypePredefined, false, nil) if err != nil { t.Fatalf("SeedUserFiles returned error: %v", err) } @@ -160,7 +178,7 @@ func TestSeedUserFiles_PredefinedAgent_FallsBackToTemplateWhenNoAgentLevelUserMD agentID := uuid.New() // No agent-level USER.md — wizard did not write one - seeded, err := SeedUserFiles(context.Background(), as, agentID, "user-bob", store.AgentTypePredefined, false) + seeded, err := SeedUserFiles(context.Background(), as, agentID, "user-bob", store.AgentTypePredefined, false, nil) if err != nil { t.Fatalf("SeedUserFiles returned error: %v", err) } @@ -198,7 +216,7 @@ func TestSeedUserFiles_PredefinedAgent_DoesNotOverwriteExistingPerUserContent(t // Also set wizard content at agent level as.agentFiles[UserFile] = "wizard content that should NOT override personal content" - seeded, err := SeedUserFiles(context.Background(), as, agentID, "user-charlie", store.AgentTypePredefined, false) + seeded, err := SeedUserFiles(context.Background(), as, agentID, "user-charlie", store.AgentTypePredefined, false, nil) if err != nil { t.Fatalf("SeedUserFiles returned error: %v", err) } @@ -223,7 +241,7 @@ func TestSeedUserFiles_OpenAgent_UsesEmbeddedTemplate(t *testing.T) { agentID := uuid.New() // Open agents should never check agent_context_files for USER.md - seeded, err := SeedUserFiles(context.Background(), as, agentID, "user-dave", store.AgentTypeOpen, false) + seeded, err := SeedUserFiles(context.Background(), as, agentID, "user-dave", store.AgentTypeOpen, false, nil) if err != nil { t.Fatalf("SeedUserFiles returned error: %v", err) } @@ -256,14 +274,14 @@ func TestSeedUserFiles_IdempotentOnSecondCall(t *testing.T) { agentID := uuid.New() // First call — seeds files - SeedUserFiles(context.Background(), as, agentID, "user-frank", store.AgentTypePredefined, false) + SeedUserFiles(context.Background(), as, agentID, "user-frank", store.AgentTypePredefined, false, nil) // Simulate what the first call wrote (move seededUserFiles → userFiles) maps.Copy(as.userFiles, as.seededUserFiles) as.seededUserFiles = make(map[string]string) // Second call — must seed nothing (all files already exist) - seeded, err := SeedUserFiles(context.Background(), as, agentID, "user-frank", store.AgentTypePredefined, false) + seeded, err := SeedUserFiles(context.Background(), as, agentID, "user-frank", store.AgentTypePredefined, false, nil) if err != nil { t.Fatalf("second SeedUserFiles returned error: %v", err) } @@ -284,7 +302,7 @@ func TestSeedUserFiles_SkipIfAnyExist(t *testing.T) { agentID := uuid.New() // Step 1: Seed initial files (new user, skipIfAnyExist=false) - SeedUserFiles(context.Background(), as, agentID, "user-eve", store.AgentTypePredefined, false) + SeedUserFiles(context.Background(), as, agentID, "user-eve", store.AgentTypePredefined, false, nil) maps.Copy(as.userFiles, as.seededUserFiles) as.seededUserFiles = make(map[string]string) @@ -292,7 +310,7 @@ func TestSeedUserFiles_SkipIfAnyExist(t *testing.T) { delete(as.userFiles, BootstrapFile) // Step 3: skipIfAnyExist=true (existing profile) → must NOT re-seed - seeded, err := SeedUserFiles(context.Background(), as, agentID, "user-eve", store.AgentTypePredefined, true) + seeded, err := SeedUserFiles(context.Background(), as, agentID, "user-eve", store.AgentTypePredefined, true, nil) if err != nil { t.Fatalf("SeedUserFiles(skipIfAnyExist=true) returned error: %v", err) } @@ -301,7 +319,7 @@ func TestSeedUserFiles_SkipIfAnyExist(t *testing.T) { } // Step 4: skipIfAnyExist=false (force) → SHOULD re-seed missing BOOTSTRAP.md - seeded, err = SeedUserFiles(context.Background(), as, agentID, "user-eve", store.AgentTypePredefined, false) + seeded, err = SeedUserFiles(context.Background(), as, agentID, "user-eve", store.AgentTypePredefined, false, nil) if err != nil { t.Fatalf("SeedUserFiles(skipIfAnyExist=false) returned error: %v", err) } @@ -323,7 +341,7 @@ func TestSeedUserFiles_SkipIfAnyExist_EmptyUser(t *testing.T) { agentID := uuid.New() // No existing user files → skipIfAnyExist=true should still seed - seeded, err := SeedUserFiles(context.Background(), as, agentID, "user-ghost", store.AgentTypePredefined, true) + seeded, err := SeedUserFiles(context.Background(), as, agentID, "user-ghost", store.AgentTypePredefined, true, nil) if err != nil { t.Fatalf("SeedUserFiles returned error: %v", err) } diff --git a/internal/cache/memory.go b/internal/cache/memory.go index 0857596d..6e673370 100644 --- a/internal/cache/memory.go +++ b/internal/cache/memory.go @@ -2,29 +2,63 @@ package cache import ( "context" + "sort" "strings" "sync" "time" ) -// entry wraps a cached value with expiration metadata. +// entry wraps a cached value with expiration and creation metadata. +// createdAt is used for oldest-first eviction when maxSize is exceeded. type entry[V any] struct { value V expiresAt time.Time // zero means no expiry + createdAt time.Time // set on Set(), used for eviction ordering } func (e entry[V]) expired() bool { return !e.expiresAt.IsZero() && time.Now().After(e.expiresAt) } -// InMemoryCache is a thread-safe in-memory Cache implementation with TTL support. +// InMemoryCache is a thread-safe in-memory Cache implementation with TTL support, +// periodic sweep goroutine for expired entries, and optional max-size cap with +// oldest-first eviction. type InMemoryCache[V any] struct { - data sync.Map + data sync.Map + maxSize int // 0 = unlimited + sweepInterval time.Duration // 0 = no periodic sweep (lazy eviction only) + cancel context.CancelFunc + closeOnce sync.Once } -// NewInMemoryCache creates a new in-memory cache. -func NewInMemoryCache[V any]() *InMemoryCache[V] { - return &InMemoryCache[V]{} +// CacheOption configures an InMemoryCache during construction. +type CacheOption[V any] func(*InMemoryCache[V]) + +// WithMaxSize sets a maximum entry count. When exceeded during sweep, the +// oldest 20% of entries are evicted. Zero = unlimited. +func WithMaxSize[V any](n int) CacheOption[V] { + return func(c *InMemoryCache[V]) { c.maxSize = n } +} + +// WithSweepInterval sets the periodic sweep interval for expired entries. +// Zero disables the sweep goroutine (lazy eviction on Get only). +func WithSweepInterval[V any](d time.Duration) CacheOption[V] { + return func(c *InMemoryCache[V]) { c.sweepInterval = d } +} + +// NewInMemoryCache creates a new in-memory cache. Without options it behaves +// exactly as before (lazy eviction, no size cap, no sweep goroutine). +func NewInMemoryCache[V any](opts ...CacheOption[V]) *InMemoryCache[V] { + c := &InMemoryCache[V]{} + for _, opt := range opts { + opt(c) + } + if c.sweepInterval > 0 { + ctx, cancel := context.WithCancel(context.Background()) + c.cancel = cancel + go c.sweepLoop(ctx) + } + return c } func (c *InMemoryCache[V]) Get(_ context.Context, key string) (V, bool) { @@ -43,7 +77,7 @@ func (c *InMemoryCache[V]) Get(_ context.Context, key string) (V, bool) { } func (c *InMemoryCache[V]) Set(_ context.Context, key string, value V, ttl time.Duration) { - e := entry[V]{value: value} + e := entry[V]{value: value, createdAt: time.Now()} if ttl > 0 { e.expiresAt = time.Now().Add(ttl) } @@ -69,3 +103,89 @@ func (c *InMemoryCache[V]) Clear(_ context.Context) { return true }) } + +// Close stops the background sweep goroutine. Safe to call multiple times. +// After Close, the cache remains readable/writable but without periodic sweep. +func (c *InMemoryCache[V]) Close() { + c.closeOnce.Do(func() { + if c.cancel != nil { + c.cancel() + } + }) +} + +// sweepLoop runs the periodic expiry + size-cap cleanup in a background goroutine. +func (c *InMemoryCache[V]) sweepLoop(ctx context.Context) { + ticker := time.NewTicker(c.sweepInterval) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + c.sweepOnce() + } + } +} + +// sweepOnce deletes expired entries and, if maxSize is set and exceeded, +// evicts the oldest 20% of remaining entries (by createdAt). +func (c *InMemoryCache[V]) sweepOnce() { + // Phase 1: collect expired keys (don't delete during Range iteration). + // sync.Map's Range is safe with concurrent Store/Delete, but deleting + // during iteration can cause the same key to be visited twice in + // pathological cases. Collecting first is cleaner. + var expiredKeys []string + type keyAge struct { + key string + createdAt time.Time + } + var allAlive []keyAge + + c.data.Range(func(k, v any) bool { + key, ok := k.(string) + if !ok { + return true + } + e, ok := v.(entry[V]) + if !ok { + return true + } + if e.expired() { + expiredKeys = append(expiredKeys, key) + } else { + allAlive = append(allAlive, keyAge{key: key, createdAt: e.createdAt}) + } + return true + }) + + for _, k := range expiredKeys { + c.data.Delete(k) + } + + // Phase 2: size-cap eviction. Sort alive entries by createdAt ascending, + // evict oldest 20% if over maxSize. + if c.maxSize > 0 && len(allAlive) > c.maxSize { + sort.Slice(allAlive, func(i, j int) bool { + return allAlive[i].createdAt.Before(allAlive[j].createdAt) + }) + toEvict := len(allAlive) - c.maxSize + (c.maxSize / 5) // bring below cap + 20% headroom + if toEvict > len(allAlive) { + toEvict = len(allAlive) + } + for i := 0; i < toEvict; i++ { + c.data.Delete(allAlive[i].key) + } + } +} + +// sizeLocked returns the current entry count (for tests and metrics). +// Named sizeLocked for historical reasons; sync.Map needs no lock. +func (c *InMemoryCache[V]) sizeLocked() int { + n := 0 + c.data.Range(func(_, _ any) bool { + n++ + return true + }) + return n +} diff --git a/internal/cache/memory_test.go b/internal/cache/memory_test.go index 44f297fd..0788372e 100644 --- a/internal/cache/memory_test.go +++ b/internal/cache/memory_test.go @@ -90,3 +90,107 @@ func TestInMemoryCache_Clear(t *testing.T) { } } } + +// TestInMemoryCache_PeriodicSweep verifies expired entries are removed by +// the background sweep goroutine (not just lazy on Get). +func TestInMemoryCache_PeriodicSweep(t *testing.T) { + c := NewInMemoryCache[string]( + WithSweepInterval[string](20*time.Millisecond), + ) + defer c.Close() + ctx := context.Background() + + c.Set(ctx, "short", "v1", 10*time.Millisecond) + c.Set(ctx, "long", "v2", 0) // no expiry + + // Wait for sweep to run at least once after short entry expires + time.Sleep(60 * time.Millisecond) + + // Short entry should be removed by sweep even without Get + if count := c.sizeLocked(); count != 1 { + t.Errorf("expected 1 entry after sweep, got %d", count) + } + if _, ok := c.Get(ctx, "long"); !ok { + t.Error("long-lived entry should still exist") + } +} + +// TestInMemoryCache_MaxSizeEviction verifies oldest entries are evicted when +// max size cap is reached. +func TestInMemoryCache_MaxSizeEviction(t *testing.T) { + c := NewInMemoryCache[int]( + WithMaxSize[int](5), + WithSweepInterval[int](10*time.Millisecond), + ) + defer c.Close() + ctx := context.Background() + + // Insert 10 entries with distinct creation times to ensure oldest-first ordering + for i := 0; i < 10; i++ { + c.Set(ctx, string(rune('a'+i)), i, 0) + time.Sleep(2 * time.Millisecond) + } + + // Trigger sweep by waiting for interval + time.Sleep(30 * time.Millisecond) + + if count := c.sizeLocked(); count > 5 { + t.Errorf("expected size ≤ 5 after max-size eviction, got %d", count) + } +} + +// TestInMemoryCache_Close verifies Close stops the sweep goroutine (no leak). +func TestInMemoryCache_Close(t *testing.T) { + c := NewInMemoryCache[string]( + WithSweepInterval[string](10*time.Millisecond), + ) + ctx := context.Background() + c.Set(ctx, "k", "v", 0) + + c.Close() + // Close should be idempotent + c.Close() + + // After Close, cache is still readable for lazy access but sweep is stopped + if _, ok := c.Get(ctx, "k"); !ok { + t.Error("Get should still work after Close") + } +} + +// TestInMemoryCache_ConcurrentSweepAndSet verifies no race between sweep and Set. +func TestInMemoryCache_ConcurrentSweepAndSet(t *testing.T) { + c := NewInMemoryCache[int]( + WithSweepInterval[int](1*time.Millisecond), + WithMaxSize[int](100), + ) + defer c.Close() + ctx := context.Background() + + done := make(chan bool) + go func() { + for i := 0; i < 500; i++ { + c.Set(ctx, string(rune('a'+(i%26))), i, 5*time.Millisecond) + } + done <- true + }() + go func() { + for i := 0; i < 500; i++ { + _, _ = c.Get(ctx, string(rune('a'+(i%26)))) + } + done <- true + }() + <-done + <-done +} + +// TestInMemoryCache_BackwardCompatZeroArg verifies existing call sites with +// zero-arg constructor still work (variadic options). +func TestInMemoryCache_BackwardCompatZeroArg(t *testing.T) { + c := NewInMemoryCache[string]() + defer c.Close() + ctx := context.Background() + c.Set(ctx, "k", "v", 0) + if v, ok := c.Get(ctx, "k"); !ok || v != "v" { + t.Errorf("backward compat broken: got %v, %v", v, ok) + } +} diff --git a/internal/cache/permission_cache.go b/internal/cache/permission_cache.go index 485a5f5a..2c0c9185 100644 --- a/internal/cache/permission_cache.go +++ b/internal/cache/permission_cache.go @@ -23,15 +23,43 @@ type PermissionCache struct { teamAccess *InMemoryCache[bool] } -// NewPermissionCache creates a new permission cache. +// permissionCacheSweepInterval and permissionCacheMaxSize bound background +// growth of per-user cache entries. Without these, long-running gateways with +// many distinct users would accumulate unbounded entries (tenant_roles, agent +// access, team access) even with a 30s TTL — lazy eviction only fires on Get, +// so entries for disconnected users never get reclaimed. +const ( + permissionCacheSweepInterval = 60 * time.Second + permissionCacheMaxSize = 10_000 +) + +// NewPermissionCache creates a new permission cache with periodic sweep +// goroutines for all three inner caches. Call Close() on gateway shutdown to +// stop the sweep goroutines. func NewPermissionCache() *PermissionCache { return &PermissionCache{ - tenantRole: NewInMemoryCache[string](), - agentAccess: NewInMemoryCache[agentAccessEntry](), - teamAccess: NewInMemoryCache[bool](), + tenantRole: NewInMemoryCache[string]( + WithSweepInterval[string](permissionCacheSweepInterval), + WithMaxSize[string](permissionCacheMaxSize), + ), + agentAccess: NewInMemoryCache[agentAccessEntry]( + WithSweepInterval[agentAccessEntry](permissionCacheSweepInterval), + WithMaxSize[agentAccessEntry](permissionCacheMaxSize), + ), + teamAccess: NewInMemoryCache[bool]( + WithSweepInterval[bool](permissionCacheSweepInterval), + WithMaxSize[bool](permissionCacheMaxSize), + ), } } +// Close stops all background sweep goroutines. Safe to call multiple times. +func (pc *PermissionCache) Close() { + pc.tenantRole.Close() + pc.agentAccess.Close() + pc.teamAccess.Close() +} + const ( tenantRoleTTL = 30 * time.Second agentAccessTTL = 30 * time.Second diff --git a/internal/channels/channel.go b/internal/channels/channel.go index e223f36f..fbdfe126 100644 --- a/internal/channels/channel.go +++ b/internal/channels/channel.go @@ -70,10 +70,12 @@ const ( // Channel type constants used across channel packages and gateway wiring. const ( - TypeTelegram = "telegram" TypeDiscord = "discord" - TypeSlack = "slack" + TypeFacebook = "facebook" TypeFeishu = "feishu" + TypePancake = "pancake" + TypeSlack = "slack" + TypeTelegram = "telegram" TypeWhatsApp = "whatsapp" TypeZaloOA = "zalo_oa" TypeZaloPersonal = "zalo_personal" diff --git a/internal/channels/discord/handler.go b/internal/channels/discord/handler.go index 7848690f..beec7555 100644 --- a/internal/channels/discord/handler.go +++ b/internal/channels/discord/handler.go @@ -264,7 +264,7 @@ func (c *Channel) handleMessage(_ *discordgo.Session, m *discordgo.MessageCreate "message_id": m.ID, "user_id": senderID, "username": m.Author.Username, - "display_name": senderName, + "display_name": channels.SanitizeDisplayName(senderName), "guild_id": m.GuildID, "channel_id": channelID, "is_dm": fmt.Sprintf("%t", isDM), diff --git a/internal/channels/dispatch.go b/internal/channels/dispatch.go index 4d2030bb..73d6c1f6 100644 --- a/internal/channels/dispatch.go +++ b/internal/channels/dispatch.go @@ -71,6 +71,9 @@ func (m *Manager) dispatchOutbound(ctx context.Context) { if err := channel.Send(ctx, msg); err != nil { slog.Error("error sending message to channel", "channel", msg.Channel, + "chat_id", msg.ChatID, + "content_len", len(msg.Content), + "content_preview", Truncate(msg.Content, 160), "error", err, ) // Try to send a text-only error notification back to the chat. diff --git a/internal/channels/facebook/comment_handler.go b/internal/channels/facebook/comment_handler.go new file mode 100644 index 00000000..5a5bc311 --- /dev/null +++ b/internal/channels/facebook/comment_handler.go @@ -0,0 +1,99 @@ +package facebook + +import ( + "fmt" + "log/slog" + "strings" +) + +// handleCommentEvent processes a feed webhook change where item == "comment". +func (ch *Channel) handleCommentEvent(entry WebhookEntry, change ChangeValue) { + // Feature gate. + if !ch.config.Features.CommentReply { + return + } + + // Only process new comments (not edits or deletions). + if change.Verb != "add" { + return + } + + // Page routing: ensure this event belongs to our page instance (before dedup write). + if entry.ID != ch.pageID { + return + } + + // Self-reply prevention: skip comments posted by the page itself. + if change.From.ID == ch.pageID { + return + } + + // Dedup: Facebook may deliver the same event more than once. + if ch.isDup("comment:" + change.CommentID) { + slog.Debug("facebook: duplicate comment event skipped", "comment_id", change.CommentID) + return + } + + // Build message content — optionally enriched with post + thread context. + // Use stopCtx so inflight Graph API calls are cancelled when the channel stops. + content := change.Message + if ch.config.CommentReplyOptions.IncludePostContext && change.PostID != "" { + content = ch.buildEnrichedContent(change) + } + + senderID := change.From.ID + // Session key groups all comments by the same user on the same post. + chatID := fmt.Sprintf("%s:%s", change.PostID, senderID) + + metadata := map[string]string{ + "comment_id": change.CommentID, + "post_id": change.PostID, + "parent_id": change.ParentID, + "sender_name": change.From.Name, + "sender_id": senderID, + "fb_mode": "comment", + "reply_to_comment_id": change.CommentID, + } + + ch.HandleMessage(senderID, chatID, content, nil, metadata, "direct") +} + +// buildEnrichedContent fetches post content and comment thread, assembles a rich +// context string for the agent. Uses stopCtx so it is cancelled on channel Stop(). +func (ch *Channel) buildEnrichedContent(change ChangeValue) string { + ctx := ch.stopCtx + + var sb strings.Builder + + // Post content. + post, err := ch.postFetcher.GetPost(ctx, change.PostID) + if err == nil && post != nil && post.Message != "" { + sb.WriteString("[Bài đăng] ") + sb.WriteString(post.Message) + sb.WriteString("\n\n") + } + + // Comment thread (when this is a nested reply). + if change.ParentID != "" { + depth := ch.config.CommentReplyOptions.MaxThreadDepth + if depth <= 0 { + depth = 10 + } + thread, err := ch.postFetcher.GetCommentThread(ctx, change.ParentID, depth) + if err == nil && len(thread) > 0 { + sb.WriteString("[Thread]\n") + for _, c := range thread { + sb.WriteString(fmt.Sprintf("- %s: %s\n", c.From.Name, c.Message)) + } + sb.WriteString("\n") + } + } + + // Current comment. + sb.WriteString("[Comment mới] ") + sb.WriteString(change.From.Name) + sb.WriteString(": ") + sb.WriteString(change.Message) + + return sb.String() +} diff --git a/internal/channels/facebook/facebook.go b/internal/channels/facebook/facebook.go new file mode 100644 index 00000000..71d4d4f7 --- /dev/null +++ b/internal/channels/facebook/facebook.go @@ -0,0 +1,260 @@ +package facebook + +import ( + "context" + "encoding/json" + "fmt" + "log/slog" + "net/http" + "sync" + "time" + + "github.com/nextlevelbuilder/goclaw/internal/bus" + "github.com/nextlevelbuilder/goclaw/internal/channels" + "github.com/nextlevelbuilder/goclaw/internal/store" +) + +// Compile-time interface assertions. +var ( + _ channels.Channel = (*Channel)(nil) + _ channels.WebhookChannel = (*Channel)(nil) +) + +const ( + webhookPath = "/channels/facebook/webhook" + dedupTTL = 24 * time.Hour // matches Facebook's max retry window + dedupCleanEvery = 5 * time.Minute // how often to evict stale dedup entries +) + +// Channel implements channels.Channel and channels.WebhookChannel for Facebook Fanpage. +// Supports comment auto-reply, Messenger auto-reply, and first inbox DM. +type Channel struct { + *channels.BaseChannel + config facebookInstanceConfig + graphClient *GraphClient + webhookH *WebhookHandler + pageID string + + // dedup prevents processing duplicate webhook deliveries. + // Value is the time.Time the event was first seen; entries are evicted after dedupTTL. + dedup sync.Map // eventKey(string) → time.Time + + // firstInboxSent tracks which senders have already received a first-inbox DM (in-memory). + firstInboxSent sync.Map // senderID(string) → struct{} + + // postFetcher caches post content to enrich comment context. + postFetcher *PostFetcher + + // stopCh + stopCtx serve complementary roles: + // stopCh for select{} in goroutines (runDedupCleaner), stopCtx for HTTP client cancellation. + stopCh chan struct{} + stopCtx context.Context + stopFn context.CancelFunc +} + +// New creates a Facebook channel from parsed credentials and config. +func New(cfg facebookInstanceConfig, creds facebookCreds, + msgBus *bus.MessageBus, _ store.PairingStore) (*Channel, error) { + + if creds.PageAccessToken == "" { + return nil, fmt.Errorf("facebook: page_access_token is required") + } + if cfg.PageID == "" { + return nil, fmt.Errorf("facebook: page_id is required") + } + if creds.AppSecret == "" { + return nil, fmt.Errorf("facebook: app_secret is required") + } + if creds.VerifyToken == "" { + return nil, fmt.Errorf("facebook: verify_token is required") + } + + base := channels.NewBaseChannel(channels.TypeFacebook, msgBus, cfg.AllowFrom) + + graphClient := NewGraphClient(creds.PageAccessToken, cfg.PageID) + postFetcher := NewPostFetcher(graphClient, cfg.PostContextCacheTTL) + + stopCtx, stopFn := context.WithCancel(context.Background()) + + ch := &Channel{ + BaseChannel: base, + config: cfg, + graphClient: graphClient, + pageID: cfg.PageID, + postFetcher: postFetcher, + stopCh: make(chan struct{}), + stopCtx: stopCtx, + stopFn: stopFn, + } + + wh := NewWebhookHandler(creds.AppSecret, creds.VerifyToken) + wh.onComment = ch.handleCommentEvent + wh.onMessage = ch.handleMessagingEvent + ch.webhookH = wh + + return ch, nil +} + +// Factory creates a Facebook Channel from DB instance data. +// Implements channels.ChannelFactory. +func Factory(name string, creds json.RawMessage, cfg json.RawMessage, + msgBus *bus.MessageBus, pairingSvc store.PairingStore) (channels.Channel, error) { + + var c facebookCreds + if err := json.Unmarshal(creds, &c); err != nil { + return nil, fmt.Errorf("facebook: decode credentials: %w", err) + } + + var ic facebookInstanceConfig + if len(cfg) > 0 { + if err := json.Unmarshal(cfg, &ic); err != nil { + return nil, fmt.Errorf("facebook: decode config: %w", err) + } + } + + ch, err := New(ic, c, msgBus, pairingSvc) + if err != nil { + return nil, err + } + ch.SetName(name) + return ch, nil +} + +// Start connects the channel: verifies the page token, subscribes webhooks, marks healthy. +func (ch *Channel) Start(ctx context.Context) error { + ch.MarkStarting("connecting to Facebook page") + + if err := ch.graphClient.VerifyToken(ctx); err != nil { + ch.MarkFailed("token invalid", err.Error(), channels.ChannelFailureKindAuth, false) + return err + } + + // Best-effort: subscribe app to webhooks. + if err := ch.graphClient.SubscribeApp(ctx); err != nil { + slog.Warn("facebook: webhook subscription failed (check app install on page)", "err", err) + } + + globalRouter.register(ch) + ch.MarkHealthy("connected to page " + ch.pageID) + ch.SetRunning(true) + + // Background goroutine: evict stale dedup entries to prevent memory growth. + go ch.runDedupCleaner() + + slog.Info("facebook channel started", "page_id", ch.pageID, "name", ch.Name()) + return nil +} + +// Stop gracefully shuts down the channel. +func (ch *Channel) Stop(_ context.Context) error { + globalRouter.unregister(ch.pageID) + ch.stopFn() // cancel stopCtx → cancels inflight Graph API calls + close(ch.stopCh) // stop background goroutines + ch.SetRunning(false) + ch.MarkStopped("stopped") + slog.Info("facebook channel stopped", "page_id", ch.pageID, "name", ch.Name()) + return nil +} + +// Send delivers an outbound message. Dispatches to comment reply or Messenger based on fb_mode metadata. +func (ch *Channel) Send(ctx context.Context, msg bus.OutboundMessage) error { + mode := msg.Metadata["fb_mode"] + + switch mode { + case "messenger": + text := FormatForMessenger(msg.Content) + parts := splitMessage(text, messengerMaxChars) + for _, part := range parts { + if _, err := ch.graphClient.SendMessage(ctx, msg.ChatID, part); err != nil { + ch.handleAPIError(err) + return err + } + } + + default: // "comment" + commentID := msg.Metadata["reply_to_comment_id"] + if commentID == "" { + return fmt.Errorf("facebook: reply_to_comment_id missing in outbound metadata") + } + text := FormatForComment(msg.Content) + if _, err := ch.graphClient.ReplyToComment(ctx, commentID, text); err != nil { + ch.handleAPIError(err) + return err + } + + // First inbox: send a private DM after comment reply (best-effort). + if ch.config.Features.FirstInbox { + senderID := msg.Metadata["sender_id"] + if senderID != "" { + ch.sendFirstInbox(ctx, senderID) + } + } + } + + return nil +} + +// WebhookHandler returns the shared webhook path and the global router as handler. +// Only the first facebook instance mounts the route; others return ("", nil). +func (ch *Channel) WebhookHandler() (string, http.Handler) { + return globalRouter.webhookRoute() +} + +// handleAPIError maps Graph API errors to channel health states. +func (ch *Channel) handleAPIError(err error) { + if err == nil { + return + } + switch { + case IsAuthError(err): + ch.MarkFailed("token expired", err.Error(), channels.ChannelFailureKindAuth, false) + case IsPermissionError(err): + ch.MarkFailed("permission denied", err.Error(), channels.ChannelFailureKindAuth, false) + case IsRateLimitError(err): + ch.MarkDegraded("rate limited", err.Error(), channels.ChannelFailureKindNetwork, true) + default: + ch.MarkDegraded("api error", err.Error(), channels.ChannelFailureKindUnknown, true) + } +} + +// sendFirstInbox sends a one-time private DM after a comment reply (best-effort). +func (ch *Channel) sendFirstInbox(ctx context.Context, senderID string) { + if _, alreadySent := ch.firstInboxSent.LoadOrStore(senderID, struct{}{}); alreadySent { + return + } + message := ch.config.FirstInboxMessage + if message == "" { + message = "Cảm ơn bạn đã comment! Mình có thể hỗ trợ thêm qua tin nhắn riêng." + } + if _, err := ch.graphClient.SendMessage(ctx, senderID, message); err != nil { + slog.Warn("facebook: first inbox send failed", "sender_id", senderID, "err", err) + ch.firstInboxSent.Delete(senderID) // allow retry on next comment + } +} + +// runDedupCleaner evicts dedup entries older than dedupTTL every dedupCleanEvery. +func (ch *Channel) runDedupCleaner() { + ticker := time.NewTicker(dedupCleanEvery) + defer ticker.Stop() + for { + select { + case <-ch.stopCh: + return + case <-ticker.C: + now := time.Now() + ch.dedup.Range(func(k, v any) bool { + if t, ok := v.(time.Time); ok && now.Sub(t) > dedupTTL { + ch.dedup.Delete(k) + } + return true + }) + } + } +} + +// isDup checks and records a dedup key. Returns true if the key was already seen. +func (ch *Channel) isDup(key string) bool { + _, loaded := ch.dedup.LoadOrStore(key, time.Now()) + return loaded +} + diff --git a/internal/channels/facebook/facebook_test.go b/internal/channels/facebook/facebook_test.go new file mode 100644 index 00000000..e915559c --- /dev/null +++ b/internal/channels/facebook/facebook_test.go @@ -0,0 +1,395 @@ +package facebook + +import ( + "crypto/hmac" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" +) + +// --- verifySignature --- + +func TestVerifySignature(t *testing.T) { + secret := "test_secret" + body := []byte(`{"object":"page","entry":[]}`) + + mac := hmac.New(sha256.New, []byte(secret)) + mac.Write(body) + validSig := "sha256=" + hex.EncodeToString(mac.Sum(nil)) + + tests := []struct { + name string + sig string + wantValid bool + }{ + {"valid signature", validSig, true}, + {"wrong prefix", "sha1=" + hex.EncodeToString(mac.Sum(nil)), false}, + {"empty signature", "", false}, + {"bad hex", "sha256=notvalidhex", false}, + {"tampered body", "sha256=0000000000000000000000000000000000000000000000000000000000000000", false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := verifySignature(body, tt.sig, secret) + if got != tt.wantValid { + t.Errorf("verifySignature() = %v, want %v", got, tt.wantValid) + } + }) + } +} + +// --- WebhookHandler GET verification --- + +func TestWebhookHandlerVerification(t *testing.T) { + wh := NewWebhookHandler("secret", "my_verify_token") + + t.Run("valid challenge", func(t *testing.T) { + req := httptest.NewRequest(http.MethodGet, + "/webhook?hub.mode=subscribe&hub.verify_token=my_verify_token&hub.challenge=abc123", nil) + w := httptest.NewRecorder() + wh.ServeHTTP(w, req) + if w.Code != http.StatusOK { + t.Fatalf("status = %d, want 200", w.Code) + } + if w.Body.String() != "abc123" { + t.Errorf("body = %q, want %q", w.Body.String(), "abc123") + } + }) + + t.Run("wrong verify token", func(t *testing.T) { + req := httptest.NewRequest(http.MethodGet, + "/webhook?hub.mode=subscribe&hub.verify_token=wrong&hub.challenge=abc123", nil) + w := httptest.NewRecorder() + wh.ServeHTTP(w, req) + if w.Code != http.StatusForbidden { + t.Errorf("status = %d, want 403", w.Code) + } + }) + + t.Run("wrong hub.mode", func(t *testing.T) { + req := httptest.NewRequest(http.MethodGet, + "/webhook?hub.mode=unsubscribe&hub.verify_token=my_verify_token&hub.challenge=abc123", nil) + w := httptest.NewRecorder() + wh.ServeHTTP(w, req) + if w.Code != http.StatusForbidden { + t.Errorf("status = %d, want 403", w.Code) + } + }) + + t.Run("unsafe challenge rejected", func(t *testing.T) { + req := httptest.NewRequest(http.MethodGet, + "/webhook?hub.mode=subscribe&hub.verify_token=my_verify_token&hub.challenge=", nil) + w := httptest.NewRecorder() + wh.ServeHTTP(w, req) + if w.Code != http.StatusBadRequest { + t.Errorf("status = %d, want 400", w.Code) + } + }) +} + +// --- WebhookHandler POST event delivery --- + +func signBody(t *testing.T, body []byte, secret string) string { + t.Helper() + mac := hmac.New(sha256.New, []byte(secret)) + mac.Write(body) + return "sha256=" + hex.EncodeToString(mac.Sum(nil)) +} + +func TestWebhookHandlerPostEvents(t *testing.T) { + appSecret := "app_secret_123" + + t.Run("invalid signature returns 200 (no retry)", func(t *testing.T) { + wh := NewWebhookHandler(appSecret, "token") + body := []byte(`{"object":"page","entry":[]}`) + req := httptest.NewRequest(http.MethodPost, "/webhook", strings.NewReader(string(body))) + req.Header.Set("X-Hub-Signature-256", "sha256=badbadbadbad") + w := httptest.NewRecorder() + wh.ServeHTTP(w, req) + if w.Code != http.StatusOK { + t.Errorf("status = %d, want 200 (Facebook must stop retrying)", w.Code) + } + }) + + t.Run("valid comment event dispatches callback", func(t *testing.T) { + var gotChange ChangeValue + wh := NewWebhookHandler(appSecret, "token") + wh.onComment = func(entry WebhookEntry, change ChangeValue) { + gotChange = change + } + + payload := WebhookPayload{ + Object: "page", + Entry: []WebhookEntry{{ + ID: "111", + Time: 1234567890, + Changes: []WebhookChange{{ + Field: "feed", + Value: ChangeValue{ + From: FBUser{ID: "456", Name: "Alice"}, + Item: "comment", + CommentID: "789", + PostID: "post1", + Message: "Hello page!", + Verb: "add", + }, + }}, + }}, + } + body, _ := json.Marshal(payload) + req := httptest.NewRequest(http.MethodPost, "/webhook", strings.NewReader(string(body))) + req.Header.Set("X-Hub-Signature-256", signBody(t, body, appSecret)) + req.Header.Set("Content-Type", "application/json") + + w := httptest.NewRecorder() + wh.ServeHTTP(w, req) + + if w.Code != http.StatusOK { + t.Fatalf("status = %d, want 200", w.Code) + } + if gotChange.CommentID != "789" { + t.Errorf("comment callback not called or wrong comment_id: %+v", gotChange) + } + }) + + t.Run("valid messenger event dispatches callback", func(t *testing.T) { + var gotEvent MessagingEvent + wh := NewWebhookHandler(appSecret, "token") + wh.onMessage = func(_ WebhookEntry, event MessagingEvent) { + gotEvent = event + } + + mid := "m_abc123" + payload := WebhookPayload{ + Object: "page", + Entry: []WebhookEntry{{ + ID: "111", + Time: 1234567890, + Messaging: []MessagingEvent{{ + Sender: FBUser{ID: "user1"}, + Recipient: FBUser{ID: "111"}, + Timestamp: 1234567890, + Message: &IncomingMessage{MID: mid, Text: "hi"}, + }}, + }}, + } + body, _ := json.Marshal(payload) + req := httptest.NewRequest(http.MethodPost, "/webhook", strings.NewReader(string(body))) + req.Header.Set("X-Hub-Signature-256", signBody(t, body, appSecret)) + + w := httptest.NewRecorder() + wh.ServeHTTP(w, req) + + if w.Code != http.StatusOK { + t.Fatalf("status = %d, want 200", w.Code) + } + if gotEvent.Message == nil || gotEvent.Message.MID != mid { + t.Errorf("messenger callback not called: %+v", gotEvent) + } + }) + + t.Run("non-page object ignored", func(t *testing.T) { + called := false + wh := NewWebhookHandler(appSecret, "token") + wh.onComment = func(_ WebhookEntry, _ ChangeValue) { called = true } + + payload := map[string]any{"object": "user", "entry": []any{}} + body, _ := json.Marshal(payload) + req := httptest.NewRequest(http.MethodPost, "/webhook", strings.NewReader(string(body))) + req.Header.Set("X-Hub-Signature-256", signBody(t, body, appSecret)) + + w := httptest.NewRecorder() + wh.ServeHTTP(w, req) + + if called { + t.Error("callback should not be called for non-page object") + } + }) +} + +// --- Formatter --- + +func TestFormatForComment(t *testing.T) { + tests := []struct { + input string + want string + }{ + {"**bold** text", "bold text"}, + {"*italic* text", "italic text"}, + {"**bold** and *italic*", "bold and italic"}, + {"[link](https://example.com)", "link (https://example.com)"}, + {"# Heading\nbody", "Heading\nbody"}, + {"html", "html"}, + {"`code`", "code"}, + // Long text truncated at 8000 chars. + {strings.Repeat("a", 9000), strings.Repeat("a", 8000)}, + } + for _, tt := range tests { + got := FormatForComment(tt.input) + if got != tt.want { + t.Errorf("FormatForComment(%q) = %q, want %q", tt.input, got, tt.want) + } + } +} + +func TestFormatForMessenger(t *testing.T) { + tests := []struct { + input string + want string + }{ + {"**bold**", "bold"}, + {"*italic*", "italic"}, + {"[link](https://x.com)", "link (https://x.com)"}, + {" leading space ", "leading space"}, + } + for _, tt := range tests { + got := FormatForMessenger(tt.input) + if got != tt.want { + t.Errorf("FormatForMessenger(%q) = %q, want %q", tt.input, got, tt.want) + } + } +} + +func TestSplitMessage(t *testing.T) { + t.Run("short message not split", func(t *testing.T) { + parts := splitMessage("hello", 2000) + if len(parts) != 1 || parts[0] != "hello" { + t.Errorf("unexpected parts: %v", parts) + } + }) + + t.Run("splits at sentence boundary", func(t *testing.T) { + long := strings.Repeat("word ", 200) + ". " + strings.Repeat("word ", 200) + parts := splitMessage(long, 500) + if len(parts) < 2 { + t.Errorf("expected multiple parts, got %d", len(parts)) + } + for _, p := range parts { + if len([]rune(p)) > 500 { + t.Errorf("part exceeds maxChars: len=%d", len([]rune(p))) + } + } + }) + + t.Run("hard cut when no boundary found", func(t *testing.T) { + noSpace := strings.Repeat("a", 3000) + parts := splitMessage(noSpace, 2000) + if len(parts) != 2 { + t.Errorf("expected 2 parts, got %d", len(parts)) + } + }) +} + +// --- validateFBID --- + +func TestValidateFBID(t *testing.T) { + valid := []string{"123456789", "123_456", "1"} + invalid := []string{"", "../me", "123abc", "123 456", "../../accounts"} + + for _, id := range valid { + if err := validateFBID(id); err != nil { + t.Errorf("validateFBID(%q) unexpectedly returned error: %v", id, err) + } + } + for _, id := range invalid { + if err := validateFBID(id); err == nil { + t.Errorf("validateFBID(%q) should have returned error", id) + } + } +} + +// --- parseRetryAfter --- + +func TestParseRetryAfter(t *testing.T) { + tests := []struct { + header string + want time.Duration + }{ + {"", 5 * time.Second}, + {"0", 5 * time.Second}, + {"notanumber", 5 * time.Second}, + {"10", 10 * time.Second}, + {"60", 60 * time.Second}, + // Capped at maxRetryAfterSec (60). + {"3600", 60 * time.Second}, + {"999999", 60 * time.Second}, + } + for _, tt := range tests { + resp := &http.Response{Header: make(http.Header)} + if tt.header != "" { + resp.Header.Set("Retry-After", tt.header) + } + got := parseRetryAfter(resp) + if got != tt.want { + t.Errorf("parseRetryAfter(%q) = %v, want %v", tt.header, got, tt.want) + } + } +} + +// --- Dedup eviction --- + +func TestDedupEviction(t *testing.T) { + // Override TTL to a tiny value so we can test eviction synchronously. + const shortTTL = 10 * time.Millisecond + + ch := &Channel{} + + key := "comment:test_event" + // First call: not a dup. + ch.dedup.Store(key, time.Now().Add(-shortTTL-time.Millisecond)) + + // Simulate cleaner: evict entries older than shortTTL. + now := time.Now() + ch.dedup.Range(func(k, v any) bool { + if t2, ok := v.(time.Time); ok && now.Sub(t2) > shortTTL { + ch.dedup.Delete(k) + } + return true + }) + + // After eviction, isDup should return false (entry gone). + if ch.isDup(key) { + // isDup stores the key now, so second call would be dup — just verify the first returns false. + // Since isDup uses LoadOrStore, a successful store means it wasn't there. + // The test above stored, deleted, so isDup should do a fresh store → return false. + } + // Verify second call IS a dup. + if !ch.isDup(key) { + t.Error("second isDup call should return true (key already stored)") + } +} + +// --- graphAPIError classification --- + +func TestErrorClassifiers(t *testing.T) { + authErr := &graphAPIError{code: 190, msg: "invalid token"} + permErr := &graphAPIError{code: 10, msg: "permission denied"} + rateErr := &graphAPIError{code: 4, msg: "rate limit"} + otherErr := &graphAPIError{code: 999, msg: "unknown"} + wrappedAuth := fmt.Errorf("send failed: %w", authErr) + + if !IsAuthError(authErr) { + t.Error("IsAuthError should be true for code 190") + } + if !IsAuthError(wrappedAuth) { + t.Error("IsAuthError should work with wrapped errors") + } + if !IsPermissionError(permErr) { + t.Error("IsPermissionError should be true for code 10") + } + if !IsRateLimitError(rateErr) { + t.Error("IsRateLimitError should be true for code 4") + } + if IsAuthError(otherErr) || IsPermissionError(otherErr) || IsRateLimitError(otherErr) { + t.Error("unknown error should not match any classifier") + } + if IsAuthError(nil) { + t.Error("nil should not be auth error") + } +} diff --git a/internal/channels/facebook/formatter.go b/internal/channels/facebook/formatter.go new file mode 100644 index 00000000..2970f016 --- /dev/null +++ b/internal/channels/facebook/formatter.go @@ -0,0 +1,104 @@ +package facebook + +import ( + "regexp" + "strings" +) + +const ( + commentMaxChars = 8000 + messengerMaxChars = 2000 +) + +var ( + // Two-pass bold: **text** first, then *text*, to avoid stray asterisk artifacts. + reBoldDouble = regexp.MustCompile(`\*\*([^*]+)\*\*`) + reBoldSingle = regexp.MustCompile(`\*([^*]+)\*`) + // markdownLink converts [text](url) → "text (url)". + reLink = regexp.MustCompile(`\[([^\]]+)\]\(([^)]+)\)`) + // HTML tags. + reHTML = regexp.MustCompile(`<[^>]+>`) + // Heading markers (# … at line start). + reHeader = regexp.MustCompile(`(?m)^#{1,6}\s+`) + // Inline `code` backticks. + reCode = regexp.MustCompile("`([^`]+)`") +) + +// FormatForComment sanitizes agent output for posting as a Facebook comment. +// Facebook comments do not support markdown or HTML. +func FormatForComment(text string) string { + text = reHeader.ReplaceAllString(text, "") + text = reBoldDouble.ReplaceAllString(text, "$1") // ** first + text = reBoldSingle.ReplaceAllString(text, "$1") // * second + text = reLink.ReplaceAllString(text, "$1 ($2)") + text = reCode.ReplaceAllString(text, "$1") + text = reHTML.ReplaceAllString(text, "") + text = strings.TrimSpace(text) + return truncateRunes(text, commentMaxChars) +} + +// FormatForMessenger sanitizes agent output for Messenger messages. +func FormatForMessenger(text string) string { + text = reHeader.ReplaceAllString(text, "") + text = reBoldDouble.ReplaceAllString(text, "$1") + text = reBoldSingle.ReplaceAllString(text, "$1") + text = reLink.ReplaceAllString(text, "$1 ($2)") + text = reCode.ReplaceAllString(text, "$1") + text = reHTML.ReplaceAllString(text, "") + return strings.TrimSpace(text) +} + +// splitMessage splits text into chunks of at most maxChars runes, +// preferring paragraph or sentence boundaries to avoid mid-word cuts. +// Uses plain-text splitting (not channels.ChunkMarkdown) because Facebook +// strips all markdown — fenced code block repair is unnecessary. +func splitMessage(text string, maxChars int) []string { + runes := []rune(text) // convert once + if len(runes) <= maxChars { + return []string{text} + } + + var parts []string + for len(runes) > maxChars { + chunk := string(runes[:maxChars]) + + // Try paragraph boundary. + if idx := strings.LastIndex(chunk, "\n\n"); idx > maxChars/2 { + parts = append(parts, strings.TrimSpace(chunk[:idx])) + runes = []rune(strings.TrimSpace(string(runes[idx+2:]))) + continue + } + + // Try sentence boundary. + split := false + for _, sep := range []string{". ", "! ", "? ", "\n"} { + if idx := strings.LastIndex(chunk, sep); idx > maxChars/2 { + parts = append(parts, strings.TrimSpace(chunk[:idx+1])) + runes = []rune(strings.TrimSpace(string(runes[idx+len(sep):]))) + split = true + break + } + } + if split { + continue + } + + // Hard cut at maxChars. + parts = append(parts, strings.TrimSpace(chunk)) + runes = []rune(strings.TrimSpace(string(runes[maxChars:]))) + } + + if remaining := strings.TrimSpace(string(runes)); remaining != "" { + parts = append(parts, remaining) + } + return parts +} + +// truncateRunes truncates s to at most n Unicode code points. +func truncateRunes(s string, n int) string { + runes := []rune(s) + if len(runes) <= n { + return s + } + return string(runes[:n]) +} diff --git a/internal/channels/facebook/graph_api.go b/internal/channels/facebook/graph_api.go new file mode 100644 index 00000000..ecc1be38 --- /dev/null +++ b/internal/channels/facebook/graph_api.go @@ -0,0 +1,355 @@ +package facebook + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "log/slog" + "net/http" + "regexp" + "strconv" + "time" +) + +const ( + graphAPIVersion = "v25.0" + graphAPIBase = "https://graph.facebook.com" + maxRetries = 3 + // maxRetryAfterSec caps the Retry-After sleep to prevent goroutine stalls on abnormal values. + maxRetryAfterSec = 60 +) + +// fbIDPattern validates Facebook object IDs: numeric or "{num}_{num}" form (post IDs). +var fbIDPattern = regexp.MustCompile(`^\d+(_\d+)?$`) + +// GraphClient wraps the Facebook Graph API for a single page instance. +type GraphClient struct { + httpClient *http.Client + pageAccessToken string + pageID string +} + +// NewGraphClient creates a new GraphClient for the given page. +func NewGraphClient(pageAccessToken, pageID string) *GraphClient { + return &GraphClient{ + httpClient: &http.Client{Timeout: 15 * time.Second}, + pageAccessToken: pageAccessToken, + pageID: pageID, + } +} + +// VerifyToken checks the page access token by calling GET /me. +func (g *GraphClient) VerifyToken(ctx context.Context) error { + data, err := g.doRequest(ctx, http.MethodGet, "/me?fields=id,name", nil) + if err != nil { + return fmt.Errorf("facebook: token verification failed: %w", err) + } + var result struct { + ID string `json:"id"` + Name string `json:"name"` + } + if err := json.Unmarshal(data, &result); err != nil { + return fmt.Errorf("facebook: token verification parse error: %w", err) + } + slog.Info("facebook: page token verified", "page_id", result.ID, "name", result.Name) + return nil +} + +// SubscribeApp subscribes the app to the page's webhook events. +func (g *GraphClient) SubscribeApp(ctx context.Context) error { + if err := validateFBID(g.pageID); err != nil { + return fmt.Errorf("facebook: subscribe app: %w", err) + } + path := fmt.Sprintf("/%s/subscribed_apps?subscribed_fields=feed,messages", g.pageID) + _, err := g.doRequest(ctx, http.MethodPost, path, nil) + if err != nil { + return fmt.Errorf("facebook: subscribe app failed: %w", err) + } + slog.Info("facebook: app subscribed to page webhooks", "page_id", g.pageID) + return nil +} + +// GetPost fetches a post by ID with message and story fields. +func (g *GraphClient) GetPost(ctx context.Context, postID string) (*GraphPost, error) { + if err := validateFBID(postID); err != nil { + return nil, fmt.Errorf("facebook: get post: %w", err) + } + path := fmt.Sprintf("/%s?fields=id,message,story,created_time", postID) + data, err := g.doRequest(ctx, http.MethodGet, path, nil) + if err != nil { + return nil, err + } + var post GraphPost + if err := json.Unmarshal(data, &post); err != nil { + return nil, fmt.Errorf("facebook: parse post: %w", err) + } + return &post, nil +} + +// GetComment fetches a single comment by ID. +func (g *GraphClient) GetComment(ctx context.Context, commentID string) (*GraphComment, error) { + if err := validateFBID(commentID); err != nil { + return nil, fmt.Errorf("facebook: get comment: %w", err) + } + path := fmt.Sprintf("/%s?fields=id,message,from,created_time", commentID) + data, err := g.doRequest(ctx, http.MethodGet, path, nil) + if err != nil { + return nil, err + } + var c GraphComment + if err := json.Unmarshal(data, &c); err != nil { + return nil, fmt.Errorf("facebook: parse comment: %w", err) + } + return &c, nil +} + +// GetCommentThread fetches up to limit comments under a parent comment. +func (g *GraphClient) GetCommentThread(ctx context.Context, parentCommentID string, limit int) ([]GraphComment, error) { + if err := validateFBID(parentCommentID); err != nil { + return nil, fmt.Errorf("facebook: get comment thread: %w", err) + } + if limit <= 0 { + limit = 10 + } + path := fmt.Sprintf("/%s/comments?fields=id,message,from,created_time&limit=%d", parentCommentID, limit) + data, err := g.doRequest(ctx, http.MethodGet, path, nil) + if err != nil { + return nil, err + } + var resp GraphListResponse[GraphComment] + if err := json.Unmarshal(data, &resp); err != nil { + return nil, fmt.Errorf("facebook: parse comment thread: %w", err) + } + return resp.Data, nil +} + +// ReplyToComment posts a reply to a comment. Returns the new comment ID. +func (g *GraphClient) ReplyToComment(ctx context.Context, commentID, message string) (string, error) { + if err := validateFBID(commentID); err != nil { + return "", fmt.Errorf("facebook: reply to comment: %w", err) + } + path := fmt.Sprintf("/%s/comments", commentID) + body := map[string]string{"message": message} + data, err := g.doRequest(ctx, http.MethodPost, path, body) + if err != nil { + return "", err + } + var result struct { + ID string `json:"id"` + } + if err := json.Unmarshal(data, &result); err != nil { + return "", fmt.Errorf("facebook: parse reply result: %w", err) + } + return result.ID, nil +} + +// SendMessage sends a Messenger message to the given recipient. Returns message ID. +func (g *GraphClient) SendMessage(ctx context.Context, recipientID, message string) (string, error) { + body := map[string]any{ + "recipient": map[string]string{"id": recipientID}, + "message": map[string]string{"text": message}, + } + data, err := g.doRequest(ctx, http.MethodPost, "/me/messages", body) + if err != nil { + return "", err + } + var result struct { + MessageID string `json:"message_id"` + } + if err := json.Unmarshal(data, &result); err != nil { + return "", fmt.Errorf("facebook: parse send message result: %w", err) + } + return result.MessageID, nil +} + +// SendTypingOn sends a typing indicator to the recipient (auto-off after 3s). +func (g *GraphClient) SendTypingOn(ctx context.Context, recipientID string) error { + body := map[string]any{ + "recipient": map[string]string{"id": recipientID}, + "sender_action": "typing_on", + } + _, err := g.doRequest(ctx, http.MethodPost, "/me/messages", body) + return err +} + +// doRequest executes a Graph API call with retries on transient errors. +// The page access token is passed via Authorization header (never in the URL). +func (g *GraphClient) doRequest(ctx context.Context, method, path string, body any) ([]byte, error) { + apiURL := fmt.Sprintf("%s/%s%s", graphAPIBase, graphAPIVersion, path) + + for attempt := 0; attempt < maxRetries; attempt++ { + if attempt > 0 { + backoff := time.Duration(1<= 500 && attempt < maxRetries-1 { + slog.Warn("facebook: server error, retrying", "status", resp.StatusCode, "attempt", attempt+1) + continue + } + + // Parse Graph API error envelope. + if resp.StatusCode >= 400 { + var apiErr graphErrorBody + if json.Unmarshal(respBody, &apiErr) == nil && apiErr.Error.Code != 0 { + // 24h messaging window violation — not retryable. + if apiErr.Error.Code == 551 || apiErr.Error.Subcode == 2018109 { + slog.Warn("facebook: 24h messaging window expired", "code", apiErr.Error.Code) + return nil, &graphAPIError{code: apiErr.Error.Code, msg: apiErr.Error.Message} + } + // Rate limited: sleep and retry (capped). + if resp.StatusCode == 429 && attempt < maxRetries-1 { + retryAfter := parseRetryAfter(resp) + slog.Warn("facebook: rate limited", "retry_after", retryAfter) + select { + case <-ctx.Done(): + return nil, ctx.Err() + case <-time.After(retryAfter): + } + continue + } + return nil, &graphAPIError{code: apiErr.Error.Code, msg: apiErr.Error.Message} + } + return nil, fmt.Errorf("facebook: http %d", resp.StatusCode) + } + + return respBody, nil + } + + // All attempts exhausted (only reachable when every iteration took the continue path). + return nil, fmt.Errorf("facebook: max retries exceeded") +} + +// logRateLimit parses the X-Business-Use-Case-Usage header and warns when approaching limits. +func (g *GraphClient) logRateLimit(resp *http.Response) { + usage := resp.Header.Get("X-Business-Use-Case-Usage") + if usage == "" { + return + } + var parsed map[string][]struct { + CallCount int `json:"call_count"` + } + if err := json.Unmarshal([]byte(usage), &parsed); err != nil { + return + } + for _, entries := range parsed { + for _, e := range entries { + if e.CallCount >= 95 { + slog.Warn("facebook: rate limit critical", "call_count_pct", e.CallCount, "page_id", g.pageID) + } else if e.CallCount >= 80 { + slog.Warn("facebook: rate limit warning", "call_count_pct", e.CallCount, "page_id", g.pageID) + } + } + } +} + +// parseRetryAfter extracts the Retry-After header, capped at maxRetryAfterSec. +func parseRetryAfter(resp *http.Response) time.Duration { + val := resp.Header.Get("Retry-After") + if val == "" { + return 5 * time.Second + } + secs, err := strconv.Atoi(val) + if err != nil || secs <= 0 { + return 5 * time.Second + } + if secs > maxRetryAfterSec { + secs = maxRetryAfterSec + } + return time.Duration(secs) * time.Second +} + +// validateFBID returns an error if id is not a valid Facebook object ID format. +// Facebook IDs are numeric strings, or "{num}_{num}" for post IDs. +func validateFBID(id string) error { + if id == "" { + return fmt.Errorf("empty facebook ID") + } + if !fbIDPattern.MatchString(id) { + return fmt.Errorf("invalid facebook ID format: %q", id) + } + return nil +} + +// graphAPIError is a structured error from the Facebook Graph API. +type graphAPIError struct { + code int + msg string +} + +func (e *graphAPIError) Error() string { + return fmt.Sprintf("facebook graph api error %d: %s", e.code, e.msg) +} + +// IsAuthError returns true when the error is an expired or invalid token. +func IsAuthError(err error) bool { + var ge *graphAPIError + if !errors.As(err, &ge) { + return false + } + return ge.code == 190 || ge.code == 102 +} + +// IsPermissionError returns true for permission-denied errors. +func IsPermissionError(err error) bool { + var ge *graphAPIError + if !errors.As(err, &ge) { + return false + } + return ge.code == 10 || ge.code == 200 +} + +// IsRateLimitError returns true for rate limit errors. +func IsRateLimitError(err error) bool { + var ge *graphAPIError + if !errors.As(err, &ge) { + return false + } + return ge.code == 4 || ge.code == 17 || ge.code == 32 || ge.code == 613 +} + diff --git a/internal/channels/facebook/messenger_handler.go b/internal/channels/facebook/messenger_handler.go new file mode 100644 index 00000000..6fcbd450 --- /dev/null +++ b/internal/channels/facebook/messenger_handler.go @@ -0,0 +1,70 @@ +package facebook + +import ( + "fmt" + "log/slog" +) + +// handleMessagingEvent processes a Messenger inbox event. +func (ch *Channel) handleMessagingEvent(entry WebhookEntry, event MessagingEvent) { + // Feature gate. + if !ch.config.Features.MessengerAutoReply { + return + } + + // Page routing guard (before dedup write). + if entry.ID != ch.pageID { + return + } + + // Self-message prevention: skip messages sent by the page itself. + if event.Sender.ID == ch.pageID { + return + } + + // Skip delivery/read receipts and other non-content events. + if event.Message == nil && event.Postback == nil { + return + } + + // Dedup by message MID or postback signature (include payload to reduce collision risk). + var eventKey string + switch { + case event.Message != nil: + eventKey = "msg:" + event.Message.MID + case event.Postback != nil: + eventKey = fmt.Sprintf("postback:%s:%d:%s", event.Sender.ID, event.Timestamp, event.Postback.Payload) + } + if ch.isDup(eventKey) { + slog.Debug("facebook: duplicate messaging event skipped", "key", eventKey) + return + } + + // Extract text content. + var content string + switch { + case event.Message != nil && event.Message.Text != "": + content = event.Message.Text + case event.Postback != nil: + content = event.Postback.Title + default: + // Attachment-only message — skip for now. + return + } + + senderID := event.Sender.ID + // Messenger sessions are 1:1: chatID = senderID (channel name scopes the session). + chatID := senderID + + metadata := map[string]string{ + "fb_mode": "messenger", + "message_id": eventKey, + "page_id": ch.pageID, + "sender_id": senderID, + } + if ch.config.MessengerOptions.SessionTimeout != "" { + metadata["session_timeout"] = ch.config.MessengerOptions.SessionTimeout + } + + ch.HandleMessage(senderID, chatID, content, nil, metadata, "direct") +} diff --git a/internal/channels/facebook/post_fetcher.go b/internal/channels/facebook/post_fetcher.go new file mode 100644 index 00000000..533d7ad8 --- /dev/null +++ b/internal/channels/facebook/post_fetcher.go @@ -0,0 +1,95 @@ +package facebook + +import ( + "context" + "sync" + "time" + + "golang.org/x/sync/singleflight" +) + +const defaultPostCacheTTL = 15 * time.Minute + +// postCacheEntry holds a cached post with its expiry time. +type postCacheEntry struct { + post *GraphPost + expiresAt time.Time +} + +// PostFetcher fetches and caches Facebook post content and comment threads. +// singleflight prevents redundant concurrent Graph API calls for the same post +// (cache stampede on viral posts receiving many comments simultaneously). +type PostFetcher struct { + graphClient *GraphClient + cacheTTL time.Duration + cache sync.Map // postID(string) → *postCacheEntry + sfGroup singleflight.Group // coalesces concurrent fetches for the same postID +} + +// NewPostFetcher creates a PostFetcher with the given cache TTL string (e.g. "15m"). +// Falls back to defaultPostCacheTTL if the string is empty or unparseable. +func NewPostFetcher(client *GraphClient, cacheTTLStr string) *PostFetcher { + ttl := defaultPostCacheTTL + if cacheTTLStr != "" { + if d, err := time.ParseDuration(cacheTTLStr); err == nil && d > 0 { + ttl = d + } + } + return &PostFetcher{ + graphClient: client, + cacheTTL: ttl, + } +} + +// GetPost returns the post for postID, using cache when fresh. +// Concurrent callers for the same postID share one inflight request. +func (pf *PostFetcher) GetPost(ctx context.Context, postID string) (*GraphPost, error) { + if postID == "" { + return nil, nil + } + + // Check cache first. + if v, ok := pf.cache.Load(postID); ok { + entry := v.(*postCacheEntry) + if time.Now().Before(entry.expiresAt) { + return entry.post, nil + } + pf.cache.Delete(postID) + } + + // Coalesce concurrent fetches for the same postID. + val, err, _ := pf.sfGroup.Do(postID, func() (any, error) { + // Re-check cache inside the singleflight call (another goroutine may have populated it). + if v, ok := pf.cache.Load(postID); ok { + entry := v.(*postCacheEntry) + if time.Now().Before(entry.expiresAt) { + return entry.post, nil + } + } + post, err := pf.graphClient.GetPost(ctx, postID) + if err != nil { + return nil, err + } + pf.cache.Store(postID, &postCacheEntry{ + post: post, + expiresAt: time.Now().Add(pf.cacheTTL), + }) + return post, nil + }) + if err != nil { + return nil, err + } + if val == nil { + return nil, nil + } + return val.(*GraphPost), nil +} + +// GetCommentThread fetches up to depth comments under parentCommentID. +// Not cached — thread content changes frequently as replies arrive. +func (pf *PostFetcher) GetCommentThread(ctx context.Context, parentCommentID string, depth int) ([]GraphComment, error) { + if parentCommentID == "" { + return nil, nil + } + return pf.graphClient.GetCommentThread(ctx, parentCommentID, depth) +} diff --git a/internal/channels/facebook/types.go b/internal/channels/facebook/types.go new file mode 100644 index 00000000..6977392a --- /dev/null +++ b/internal/channels/facebook/types.go @@ -0,0 +1,148 @@ +// Package facebook implements the Facebook Fanpage channel for GoClaw. +// Supports: comment auto-reply, Messenger inbox auto-reply, first inbox DM. +package facebook + +// facebookCreds holds encrypted credentials stored in channel_instances.credentials. +type facebookCreds struct { + PageAccessToken string `json:"page_access_token"` + AppSecret string `json:"app_secret"` + VerifyToken string `json:"verify_token"` +} + +// facebookInstanceConfig holds non-secret config from channel_instances.config JSONB. +type facebookInstanceConfig struct { + PageID string `json:"page_id"` + Features struct { + CommentReply bool `json:"comment_reply"` + FirstInbox bool `json:"first_inbox"` + MessengerAutoReply bool `json:"messenger_auto_reply"` + } `json:"features"` + CommentReplyOptions struct { + IncludePostContext bool `json:"include_post_context"` + MaxThreadDepth int `json:"max_thread_depth"` + } `json:"comment_reply_options"` + MessengerOptions struct { + SessionTimeout string `json:"session_timeout"` + } `json:"messenger_options"` + PostContextCacheTTL string `json:"post_context_cache_ttl"` + // FirstInboxMessage is the DM text sent to commenters (first-inbox feature). + // Defaults to Vietnamese if empty. Operators should set this to match their page language. + FirstInboxMessage string `json:"first_inbox_message,omitempty"` + AllowFrom []string `json:"allow_from,omitempty"` +} + +// --- Webhook payloads --- + +// WebhookPayload is the top-level Facebook webhook event payload. +type WebhookPayload struct { + Object string `json:"object"` + Entry []WebhookEntry `json:"entry"` +} + +// WebhookEntry is one page's events within a webhook delivery. +type WebhookEntry struct { + ID string `json:"id"` // page_id + Time int64 `json:"time"` + Changes []WebhookChange `json:"changes,omitempty"` // feed events (comments, posts) + Messaging []MessagingEvent `json:"messaging,omitempty"` // Messenger events +} + +// WebhookChange is a single change event for feed subscriptions. +type WebhookChange struct { + Field string `json:"field"` // "feed", "mention", etc. + Value ChangeValue `json:"value"` +} + +// ChangeValue holds the details of a feed change event. +type ChangeValue struct { + From FBUser `json:"from"` + Item string `json:"item"` // "comment", "post", "status" + CommentID string `json:"comment_id"` + PostID string `json:"post_id"` + ParentID string `json:"parent_id"` // parent comment ID for nested replies + Message string `json:"message"` + Verb string `json:"verb"` // "add", "edit", "remove" + CreatedTime int64 `json:"created_time"` +} + +// MessagingEvent is a single Messenger inbox event. +type MessagingEvent struct { + Sender FBUser `json:"sender"` + Recipient FBUser `json:"recipient"` + Timestamp int64 `json:"timestamp"` + Message *IncomingMessage `json:"message,omitempty"` + Postback *Postback `json:"postback,omitempty"` +} + +// IncomingMessage holds a Messenger text/attachment message. +type IncomingMessage struct { + MID string `json:"mid"` + Text string `json:"text"` + Attachments []Attachment `json:"attachments,omitempty"` +} + +// Postback holds a Messenger postback event. +type Postback struct { + Title string `json:"title"` + Payload string `json:"payload"` +} + +// Attachment is a Messenger media attachment. +type Attachment struct { + Type string `json:"type"` // "image", "video", "audio", "file" + Payload AttachmentPayload `json:"payload"` +} + +// AttachmentPayload holds the URL of a media attachment. +type AttachmentPayload struct { + URL string `json:"url"` +} + +// FBUser is a minimal Facebook user reference. +type FBUser struct { + ID string `json:"id"` + Name string `json:"name,omitempty"` +} + +// --- Graph API response types --- + +// GraphComment is a Facebook comment object from the Graph API. +type GraphComment struct { + ID string `json:"id"` + Message string `json:"message"` + From FBUser `json:"from"` + CreatedTime string `json:"created_time"` +} + +// GraphPost is a Facebook post object from the Graph API. +type GraphPost struct { + ID string `json:"id"` + Message string `json:"message"` + Story string `json:"story,omitempty"` + CreatedTime string `json:"created_time"` +} + +// GraphPaging holds cursor-based pagination for Graph API list responses. +type GraphPaging struct { + Cursors struct { + Before string `json:"before"` + After string `json:"after"` + } `json:"cursors"` + Next string `json:"next,omitempty"` +} + +// GraphListResponse is a generic Graph API list response. +type GraphListResponse[T any] struct { + Data []T `json:"data"` + Paging GraphPaging `json:"paging,omitempty"` +} + +// graphErrorBody is the error envelope returned by Graph API on failures. +type graphErrorBody struct { + Error struct { + Message string `json:"message"` + Type string `json:"type"` + Code int `json:"code"` + Subcode int `json:"error_subcode"` + } `json:"error"` +} diff --git a/internal/channels/facebook/webhook_handler.go b/internal/channels/facebook/webhook_handler.go new file mode 100644 index 00000000..85f0b1d6 --- /dev/null +++ b/internal/channels/facebook/webhook_handler.go @@ -0,0 +1,152 @@ +package facebook + +import ( + "crypto/hmac" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "io" + "log/slog" + "net/http" + "regexp" + "strings" +) + +// WebhookHandler implements http.Handler for the Facebook webhook endpoint. +// Handles both the GET verification challenge and POST event delivery. +type WebhookHandler struct { + appSecret string + verifyToken string + extraSecrets []string // additional app secrets for multi-Meta-App deployments + onComment func(entry WebhookEntry, change ChangeValue) + onMessage func(entry WebhookEntry, event MessagingEvent) +} + +// NewWebhookHandler creates a new WebhookHandler. +func NewWebhookHandler(appSecret, verifyToken string) *WebhookHandler { + return &WebhookHandler{ + appSecret: appSecret, + verifyToken: verifyToken, + } +} + +// hubChallengePattern validates that hub.challenge is safe to reflect. +var hubChallengePattern = regexp.MustCompile(`^[a-zA-Z0-9_\-]{1,256}$`) + +// ServeHTTP handles Facebook webhook GET (verification) and POST (event delivery). +func (wh *WebhookHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { + switch r.Method { + case http.MethodGet: + wh.handleVerification(w, r) + case http.MethodPost: + wh.handleEvent(w, r) + default: + http.Error(w, "method not allowed", http.StatusMethodNotAllowed) + } +} + +// handleVerification responds to Facebook's webhook verification challenge. +func (wh *WebhookHandler) handleVerification(w http.ResponseWriter, r *http.Request) { + q := r.URL.Query() + if q.Get("hub.mode") != "subscribe" { + http.Error(w, "invalid hub.mode", http.StatusForbidden) + return + } + if q.Get("hub.verify_token") != wh.verifyToken { + slog.Warn("security.facebook_webhook_verify_token_mismatch", + "remote_addr", r.RemoteAddr) + http.Error(w, "invalid verify token", http.StatusForbidden) + return + } + challenge := q.Get("hub.challenge") + // Validate challenge before reflecting to prevent injection if Content-Type changes. + if !hubChallengePattern.MatchString(challenge) { + http.Error(w, "invalid challenge", http.StatusBadRequest) + return + } + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte(challenge)) +} + +// handleEvent processes a Facebook webhook event delivery. +// Always returns 200 OK — Facebook retries on non-2xx for 24h. +func (wh *WebhookHandler) handleEvent(w http.ResponseWriter, r *http.Request) { + const maxBodyBytes = 4 << 20 // 4 MB + lr := io.LimitReader(r.Body, maxBodyBytes+1) + body, err := io.ReadAll(lr) + if err != nil { + slog.Warn("facebook: webhook read body error", "err", err) + w.WriteHeader(http.StatusOK) // 200 so Facebook stops retrying a bad delivery + return + } + if len(body) > maxBodyBytes { + slog.Warn("facebook: webhook body exceeded limit, event dropped", "bytes", len(body)) + w.WriteHeader(http.StatusOK) + return + } + + sig := r.Header.Get("X-Hub-Signature-256") + verified := verifySignature(body, sig, wh.appSecret) + if !verified { + // Try extra secrets (multi-Meta-App deployments share one webhook endpoint). + for _, s := range wh.extraSecrets { + if verifySignature(body, sig, s) { + verified = true + break + } + } + } + if !verified { + slog.Warn("security.facebook_webhook_signature_invalid", "remote_addr", r.RemoteAddr) + w.WriteHeader(http.StatusOK) // return 200 so Facebook stops sending + return + } + + var payload WebhookPayload + if err := json.Unmarshal(body, &payload); err != nil { + slog.Warn("facebook: webhook parse error", "err", err) + w.WriteHeader(http.StatusOK) + return + } + + if payload.Object != "page" { + w.WriteHeader(http.StatusOK) + return + } + + for _, entry := range payload.Entry { + // Feed events (comments, posts). + for _, change := range entry.Changes { + if change.Field == "feed" && change.Value.Item == "comment" { + if wh.onComment != nil { + wh.onComment(entry, change.Value) + } + } + } + // Messenger events. + for _, event := range entry.Messaging { + if wh.onMessage != nil { + wh.onMessage(entry, event) + } + } + } + + w.WriteHeader(http.StatusOK) +} + +// verifySignature validates the X-Hub-Signature-256 header using HMAC-SHA256. +// Facebook sends "sha256="; we recompute and compare. +func verifySignature(body []byte, signature, appSecret string) bool { + const prefix = "sha256=" + if !strings.HasPrefix(signature, prefix) { + return false + } + expected, err := hex.DecodeString(signature[len(prefix):]) + if err != nil { + return false + } + mac := hmac.New(sha256.New, []byte(appSecret)) + mac.Write(body) + computed := mac.Sum(nil) + return hmac.Equal(computed, expected) +} diff --git a/internal/channels/facebook/webhook_router.go b/internal/channels/facebook/webhook_router.go new file mode 100644 index 00000000..05cacc17 --- /dev/null +++ b/internal/channels/facebook/webhook_router.go @@ -0,0 +1,106 @@ +package facebook + +import ( + "log/slog" + "net/http" + "sync" +) + +// webhookRouter routes incoming Facebook webhook events to the correct channel instance by page_id. +// A single HTTP handler is shared across all facebook channel instances on the same server. +// +// Multi-Meta-App note: all page instances registered here are expected to share the same Meta App +// (and thus the same app_secret). If instances with different secrets are registered, ServeHTTP +// tries all known secrets and accepts the payload if any matches. +type webhookRouter struct { + mu sync.RWMutex + instances map[string]*Channel // pageID → channel + routeHandled bool // true after first webhookRoute() call +} + +var globalRouter = &webhookRouter{ + instances: make(map[string]*Channel), +} + +func (r *webhookRouter) register(ch *Channel) { + r.mu.Lock() + defer r.mu.Unlock() + r.instances[ch.pageID] = ch +} + +func (r *webhookRouter) unregister(pageID string) { + r.mu.Lock() + defer r.mu.Unlock() + delete(r.instances, pageID) +} + +// webhookRoute returns the path+handler for the first call; ("", nil) for subsequent calls. +func (r *webhookRouter) webhookRoute() (string, http.Handler) { + r.mu.Lock() + defer r.mu.Unlock() + if !r.routeHandled { + r.routeHandled = true + return webhookPath, r + } + return "", nil +} + +// ServeHTTP is the shared handler for all Facebook page webhooks. +// Routes each entry to the matching channel instance by page_id. +// +// Multi-Meta-App support: all registered page secrets are collected and tried in order. +// A payload is accepted if its signature matches any known app_secret. In the common +// case (single Meta App) there is exactly one secret and behavior is unchanged. +func (r *webhookRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) { + r.mu.RLock() + var primarySecret, verifyToken string + var extraSecrets []string + seenSecrets := make(map[string]bool) + for _, ch := range r.instances { + s := ch.webhookH.appSecret + if primarySecret == "" { + primarySecret = s + verifyToken = ch.webhookH.verifyToken + seenSecrets[s] = true + } else if !seenSecrets[s] { + extraSecrets = append(extraSecrets, s) + seenSecrets[s] = true + } + } + r.mu.RUnlock() + + if primarySecret == "" { + // No instances registered yet. + w.WriteHeader(http.StatusOK) + return + } + + if len(extraSecrets) > 0 { + slog.Warn("security.facebook_multi_meta_app", + "extra_app_count", len(extraSecrets), + "note", "multiple Meta App secrets registered; payloads verified against all known secrets") + } + + routingWH := &WebhookHandler{ + appSecret: primarySecret, + verifyToken: verifyToken, + extraSecrets: extraSecrets, + } + routingWH.onComment = func(entry WebhookEntry, change ChangeValue) { + r.mu.RLock() + target := r.instances[entry.ID] + r.mu.RUnlock() + if target != nil { + target.handleCommentEvent(entry, change) + } + } + routingWH.onMessage = func(entry WebhookEntry, event MessagingEvent) { + r.mu.RLock() + target := r.instances[entry.ID] + r.mu.RUnlock() + if target != nil { + target.handleMessagingEvent(entry, event) + } + } + routingWH.ServeHTTP(w, req) +} diff --git a/internal/channels/feishu/bot.go b/internal/channels/feishu/bot.go index 98ba7d65..4020c6a3 100644 --- a/internal/channels/feishu/bot.go +++ b/internal/channels/feishu/bot.go @@ -169,7 +169,7 @@ func (c *Channel) handleMessageEvent(ctx context.Context, event *MessageEvent) { "message_id": messageID, "chat_type": mc.ChatType, "sender_name": senderName, - "display_name": senderName, + "display_name": channels.SanitizeDisplayName(senderName), "mentioned_bot": fmt.Sprintf("%t", mc.MentionedBot), "platform": channels.TypeFeishu, } diff --git a/internal/channels/pancake/api_client.go b/internal/channels/pancake/api_client.go new file mode 100644 index 00000000..507077b4 --- /dev/null +++ b/internal/channels/pancake/api_client.go @@ -0,0 +1,240 @@ +package pancake + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "log/slog" + "mime/multipart" + "net/http" + "path/filepath" + "time" +) + +const ( + publicAPIBase = "https://pages.fm/api/public_api/v2" // page-level APIs + userAPIBase = "https://pages.fm/api/v1" // user-level APIs (list pages, etc.) + httpTimeout = 30 * time.Second +) + +// APIClient wraps the Pancake REST API for a single page instance. +type APIClient struct { + pageV1BaseURL string + pageV2BaseURL string + userBaseURL string + pageAccessToken string + apiKey string + pageID string + httpClient *http.Client +} + +// NewAPIClient creates a new Pancake APIClient for the given page. +func NewAPIClient(apiKey, pageAccessToken, pageID string) *APIClient { + return &APIClient{ + pageV1BaseURL: "https://pages.fm/api/public_api/v1", + pageV2BaseURL: publicAPIBase, + userBaseURL: userAPIBase, + pageAccessToken: pageAccessToken, + apiKey: apiKey, + pageID: pageID, + httpClient: &http.Client{Timeout: httpTimeout}, + } +} + +// VerifyToken validates the page_access_token via a lightweight API call. +func (c *APIClient) VerifyToken(ctx context.Context) error { + url := fmt.Sprintf("%s/pages/%s/conversations?limit=1", c.pageV2BaseURL, c.pageID) + if err := c.doRequest(ctx, http.MethodGet, url, nil); err != nil { + return fmt.Errorf("pancake: token verification failed: %w", err) + } + slog.Info("pancake: page token verified", "page_id", c.pageID) + return nil +} + +// GetPage fetches page metadata including platform (facebook/zalo/instagram/tiktok/whatsapp/line). +func (c *APIClient) GetPage(ctx context.Context) (*PageInfo, error) { + url := fmt.Sprintf("%s/pages", c.userBaseURL) + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + if err != nil { + return nil, fmt.Errorf("pancake: build get-pages request: %w", err) + } + req.Header.Set("Authorization", "Bearer "+c.apiKey) + req.Header.Set("Content-Type", "application/json") + + res, err := c.httpClient.Do(req) + if err != nil { + return nil, fmt.Errorf("pancake: get pages request failed: %w", err) + } + defer res.Body.Close() + + body, err := io.ReadAll(res.Body) + if err != nil { + return nil, fmt.Errorf("pancake: read get-pages response: %w", err) + } + + var result struct { + Data []PageInfo `json:"data"` + } + if err := json.Unmarshal(body, &result); err != nil { + return nil, fmt.Errorf("pancake: parse get-pages response: %w", err) + } + + for i := range result.Data { + if result.Data[i].ID == c.pageID { + return &result.Data[i], nil + } + } + + // Page not found in list — return minimal info without platform + slog.Warn("pancake: page not found in pages list, platform unknown", "page_id", c.pageID) + return &PageInfo{ID: c.pageID}, nil +} + +// SendMessage sends a text message to a conversation. +func (c *APIClient) SendMessage(ctx context.Context, conversationID, content string) error { + body, _ := json.Marshal(SendMessageRequest{ + Action: "reply_inbox", + Message: content, + }) + url := fmt.Sprintf("%s/pages/%s/conversations/%s/messages", c.pageV1BaseURL, c.pageID, conversationID) + if err := c.doRequest(ctx, http.MethodPost, url, bytes.NewReader(body)); err != nil { + return fmt.Errorf("pancake: send message: %w", err) + } + return nil +} + +// SendAttachmentMessage sends one or more uploaded content IDs to a conversation. +func (c *APIClient) SendAttachmentMessage(ctx context.Context, conversationID string, contentIDs []string) error { + body, _ := json.Marshal(SendMessageRequest{ + Action: "reply_inbox", + ContentIDs: contentIDs, + }) + url := fmt.Sprintf("%s/pages/%s/conversations/%s/messages", c.pageV1BaseURL, c.pageID, conversationID) + if err := c.doRequest(ctx, http.MethodPost, url, bytes.NewReader(body)); err != nil { + return fmt.Errorf("pancake: send attachment message: %w", err) + } + return nil +} + +// UploadMedia uploads a file via multipart/form-data and returns the attachment ID. +func (c *APIClient) UploadMedia(ctx context.Context, filename string, data io.Reader, contentType string) (string, error) { + var buf bytes.Buffer + mw := multipart.NewWriter(&buf) + + fw, err := mw.CreateFormFile("file", filepath.Base(filename)) + if err != nil { + return "", fmt.Errorf("pancake: create form file: %w", err) + } + if _, err := io.Copy(fw, data); err != nil { + return "", fmt.Errorf("pancake: copy file data: %w", err) + } + mw.Close() + + url := fmt.Sprintf("%s/pages/%s/upload_contents", c.pageV1BaseURL, c.pageID) + req, err := c.newPageRequest(ctx, http.MethodPost, url, &buf) + if err != nil { + return "", fmt.Errorf("pancake: build upload request: %w", err) + } + req.Header.Set("Content-Type", mw.FormDataContentType()) + + res, err := c.httpClient.Do(req) + if err != nil { + return "", fmt.Errorf("pancake: upload request failed: %w", err) + } + defer res.Body.Close() + + respBody, err := io.ReadAll(res.Body) + if err != nil { + return "", fmt.Errorf("pancake: read upload response: %w", err) + } + + var uploadResp UploadResponse + if err := json.Unmarshal(respBody, &uploadResp); err != nil { + return "", fmt.Errorf("pancake: parse upload response: %w", err) + } + + if uploadResp.ID == "" { + return "", fmt.Errorf("pancake: upload response missing attachment ID") + } + + return uploadResp.ID, nil +} + +// doRequest executes an authenticated HTTP request using the page_access_token. +// Always drains and closes the response body to enable connection reuse. +func (c *APIClient) doRequest(ctx context.Context, method, url string, body io.Reader) error { + req, err := c.newPageRequest(ctx, method, url, body) + if err != nil { + return err + } + req.Header.Set("Content-Type", "application/json") + + res, err := c.httpClient.Do(req) + if err != nil { + return err + } + defer res.Body.Close() + + // Always read the full body to allow HTTP connection reuse. + respBody, _ := io.ReadAll(res.Body) + + if res.StatusCode >= 400 { + var apiErr apiError + if jsonErr := json.Unmarshal(respBody, &apiErr); jsonErr == nil && apiErr.Message != "" { + return &apiErr + } + return fmt.Errorf("pancake: HTTP %d", res.StatusCode) + } + + // Some Pancake endpoints return HTTP 200 with a JSON body carrying success=false. + // Treat these as application-level send failures instead of silent success. + var appResp struct { + Success *bool `json:"success,omitempty"` + Message string `json:"message,omitempty"` + } + if err := json.Unmarshal(respBody, &appResp); err == nil && appResp.Success != nil && !*appResp.Success { + if appResp.Message != "" { + return fmt.Errorf("pancake: %s", appResp.Message) + } + return fmt.Errorf("pancake: request reported success=false") + } + + return nil +} + +func (c *APIClient) newPageRequest(ctx context.Context, method, rawURL string, body io.Reader) (*http.Request, error) { + req, err := http.NewRequestWithContext(ctx, method, rawURL, body) + if err != nil { + return nil, err + } + + query := req.URL.Query() + query.Set("page_access_token", c.pageAccessToken) + req.URL.RawQuery = query.Encode() + + // Keep the header for compatibility; official docs require the query token. + req.Header.Set("Authorization", "Bearer "+c.pageAccessToken) + return req, nil +} + +// isAuthError checks if an error is an authentication/authorization failure. +// Uses errors.As to handle wrapped errors consistently with Facebook channel pattern. +func isAuthError(err error) bool { + var ae *apiError + if !errors.As(err, &ae) { + return false + } + return ae.Code == 401 || ae.Code == 403 || ae.Code == 4001 || ae.Code == 4003 +} + +// isRateLimitError checks if an error is a rate limit response. +func isRateLimitError(err error) bool { + var ae *apiError + if !errors.As(err, &ae) { + return false + } + return ae.Code == 429 || ae.Code == 4029 +} diff --git a/internal/channels/pancake/echo_dedup.go b/internal/channels/pancake/echo_dedup.go new file mode 100644 index 00000000..2695cddd --- /dev/null +++ b/internal/channels/pancake/echo_dedup.go @@ -0,0 +1,146 @@ +package pancake + +import ( + "html" + "regexp" + "strings" + "time" +) + +var ( + htmlBreakTagRe = regexp.MustCompile(`(?i)]*\/?>`) + htmlCloseTagRe = regexp.MustCompile(`(?i)(?:div|p|li|ul|ol|h[1-6])>`) + htmlTagRe = regexp.MustCompile(`(?i)<[^>]+>`) +) + +// isDup checks and records a dedup key. Returns true if the key was already seen. +func (ch *Channel) isDup(key string) bool { + _, loaded := ch.dedup.LoadOrStore(key, time.Now()) + return loaded +} + +// splitMessage splits text into chunks no longer than maxLen Unicode code points. +// Operates on runes (not bytes) to avoid splitting multi-byte characters (CJK, emoji, Vietnamese). +func splitMessage(text string, maxLen int) []string { + runes := []rune(text) + if maxLen <= 0 || len(runes) <= maxLen { + return []string{text} + } + var parts []string + for len(runes) > maxLen { + parts = append(parts, string(runes[:maxLen])) + runes = runes[maxLen:] + } + if len(runes) > 0 { + parts = append(parts, string(runes)) + } + return parts +} + +func (ch *Channel) forgetOutboundEcho(conversationID, content string) { + if conversationID == "" { + return + } + normalized := normalizeEchoContent(content) + if normalized == "" { + return + } + ch.recentOutbound.Delete(conversationID + "\x00" + normalized) +} + +func (ch *Channel) rememberOutboundEcho(conversationID, content string) { + if conversationID == "" { + return + } + normalized := normalizeEchoContent(content) + if normalized == "" { + return + } + ch.recentOutbound.Store(conversationID+"\x00"+normalized, time.Now()) +} + +func (ch *Channel) isRecentOutboundEcho(conversationID, content string) bool { + if conversationID == "" { + return false + } + normalized := normalizeEchoContent(content) + if normalized == "" { + return false + } + key := conversationID + "\x00" + normalized + v, ok := ch.recentOutbound.Load(key) + if !ok { + return false + } + ts, ok := v.(time.Time) + if !ok { + ch.recentOutbound.Delete(key) + return false + } + if time.Since(ts) > outboundEchoTTL { + ch.recentOutbound.Delete(key) + return false + } + return true +} + +func normalizeEchoContent(content string) string { + content = strings.TrimSpace(content) + if content == "" { + return "" + } + + content = html.UnescapeString(content) + content = strings.ReplaceAll(content, "\r\n", "\n") + content = strings.ReplaceAll(content, "\r", "\n") + content = htmlBreakTagRe.ReplaceAllString(content, "\n") + content = htmlCloseTagRe.ReplaceAllString(content, "\n") + content = htmlTagRe.ReplaceAllString(content, "") + + lines := strings.Split(content, "\n") + normalized := make([]string, 0, len(lines)) + pendingBlank := false + for _, line := range lines { + line = strings.Join(strings.Fields(line), " ") + if line == "" { + if len(normalized) == 0 || pendingBlank { + continue + } + pendingBlank = true + continue + } + if pendingBlank { + normalized = append(normalized, "") + pendingBlank = false + } + normalized = append(normalized, line) + } + + return strings.TrimSpace(strings.Join(normalized, "\n")) +} + +// runDedupCleaner evicts dedup entries older than dedupTTL every dedupCleanEvery. +func (ch *Channel) runDedupCleaner() { + ticker := time.NewTicker(dedupCleanEvery) + defer ticker.Stop() + for { + select { + case <-ch.stopCh: + return + case <-ticker.C: + now := time.Now() + ch.dedup.Range(func(k, v any) bool { + if t, ok := v.(time.Time); ok && now.Sub(t) > dedupTTL { + ch.dedup.Delete(k) + } + return true + }) + ch.recentOutbound.Range(func(k, v any) bool { + if t, ok := v.(time.Time); ok && now.Sub(t) > outboundEchoTTL { + ch.recentOutbound.Delete(k) + } + return true + }) + } + } +} diff --git a/internal/channels/pancake/formatter.go b/internal/channels/pancake/formatter.go new file mode 100644 index 00000000..609c63ba --- /dev/null +++ b/internal/channels/pancake/formatter.go @@ -0,0 +1,87 @@ +package pancake + +import ( + "regexp" + "strings" +) + +// FormatOutbound formats agent response text for the target platform. +// Each platform has different formatting rules and supported markup. +func FormatOutbound(content string, platform string) string { + switch platform { + case "facebook": + return formatForFacebook(content) + case "whatsapp": + return formatForWhatsApp(content) + case "zalo", "instagram", "line": + return stripMarkdown(content) + case "tiktok": + return stripMarkdown(truncateForTikTok(content)) + default: + return stripMarkdown(content) + } +} + +// formatForFacebook allows basic HTML tags supported by Messenger. +// Strips unsupported tags, keeps bold/italic/links. +func formatForFacebook(content string) string { + // Convert markdown bold (**text** or __text__) to plain (FB Messenger uses plain text) + content = reBold.ReplaceAllString(content, "$1") + content = reItalic.ReplaceAllString(content, "$1") + // Strip markdown code blocks and inline code + content = reCodeBlock.ReplaceAllString(content, "$1") + content = reInlineCode.ReplaceAllString(content, "$1") + // Strip markdown headers (## Heading → Heading) + content = reHeader.ReplaceAllString(content, "$1") + return strings.TrimSpace(content) +} + +// formatForWhatsApp converts markdown to WhatsApp-native formatting. +// WhatsApp uses *bold*, _italic_, ~strikethrough~, ```code```. +func formatForWhatsApp(content string) string { + // Convert **bold** → *bold* (WhatsApp format) + content = reDoubleBold.ReplaceAllString(content, "*$1*") + // Convert __italic__ → _italic_ (already matches WA format, just clean up __) + content = reDoubleUnderline.ReplaceAllString(content, "_$1_") + // Strip markdown headers + content = reHeader.ReplaceAllString(content, "$1") + // Strip inline code backticks (keep content) + content = reInlineCode.ReplaceAllString(content, "$1") + return strings.TrimSpace(content) +} + +// stripMarkdown removes common markdown formatting, returning plain text. +func stripMarkdown(content string) string { + content = reBold.ReplaceAllString(content, "$1") + content = reItalic.ReplaceAllString(content, "$1") + content = reCodeBlock.ReplaceAllString(content, "$1") + content = reInlineCode.ReplaceAllString(content, "$1") + content = reHeader.ReplaceAllString(content, "$1") + content = reLink.ReplaceAllString(content, "$1") + content = reImage.ReplaceAllString(content, "") + return strings.TrimSpace(content) +} + +// truncateForTikTok truncates content to TikTok DM limit (500 runes). +// Uses rune slicing to avoid corrupting multi-byte UTF-8 (CJK, Vietnamese, emoji). +func truncateForTikTok(content string) string { + const limit = 500 + runes := []rune(content) + if len(runes) <= limit { + return content + } + return string(runes[:limit-3]) + "..." +} + +// Compiled regexes for markdown stripping — package-level for efficiency. +var ( + reBold = regexp.MustCompile(`(?:\*\*|__)(.+?)(?:\*\*|__)`) + reDoubleBold = regexp.MustCompile(`\*\*(.+?)\*\*`) + reDoubleUnderline = regexp.MustCompile(`__(.+?)__`) + reItalic = regexp.MustCompile(`(?:\*|_)(.+?)(?:\*|_)`) + reCodeBlock = regexp.MustCompile("(?s)```(?:[a-z]*)?\n?(.+?)```") + reInlineCode = regexp.MustCompile("`(.+?)`") + reHeader = regexp.MustCompile(`(?m)^#{1,6}\s+(.+)$`) + reLink = regexp.MustCompile(`\[([^\]]+)\]\([^)]+\)`) + reImage = regexp.MustCompile(`!\[[^\]]*\]\([^)]+\)`) +) diff --git a/internal/channels/pancake/media_handler.go b/internal/channels/pancake/media_handler.go new file mode 100644 index 00000000..f5a92682 --- /dev/null +++ b/internal/channels/pancake/media_handler.go @@ -0,0 +1,72 @@ +package pancake + +import ( + "context" + "fmt" + "log/slog" + "mime" + "os" + "path/filepath" + + "github.com/nextlevelbuilder/goclaw/internal/bus" +) + +// handleMediaAttachments uploads media files from an OutboundMessage and returns attachment IDs. +// Returns an empty slice (not an error) when no media is attached. +// On upload failure the error is returned so the caller can decide to send text-only. +func (ch *Channel) handleMediaAttachments(ctx context.Context, msg bus.OutboundMessage) ([]string, error) { + if len(msg.Media) == 0 { + return nil, nil + } + + var ids []string + for _, att := range msg.Media { + if att.URL == "" { + continue + } + + id, err := ch.uploadMediaFile(ctx, att.URL, att.ContentType) + if err != nil { + slog.Warn("pancake: skipping media attachment on upload error", + "page_id", ch.pageID, "url", att.URL, "err", err) + return ids, err + } + ids = append(ids, id) + } + + return ids, nil +} + +// uploadMediaFile opens a local file path and uploads it to the Pancake API. +// path must be absolute to prevent directory traversal via relative paths. +func (ch *Channel) uploadMediaFile(ctx context.Context, path string, contentType string) (string, error) { + cleanPath := filepath.Clean(path) + if !filepath.IsAbs(cleanPath) { + return "", fmt.Errorf("pancake: media path must be absolute, got: %s", path) + } + f, err := os.Open(cleanPath) + if err != nil { + return "", err + } + defer f.Close() + + ct := contentType + if ct == "" { + ct = mimeTypeFromPath(cleanPath) + } + + return ch.apiClient.UploadMedia(ctx, filepath.Base(cleanPath), f, ct) +} + +// mimeTypeFromPath guesses the MIME type from the file extension. +func mimeTypeFromPath(path string) string { + ext := filepath.Ext(path) + if ext == "" { + return "application/octet-stream" + } + mt := mime.TypeByExtension(ext) + if mt == "" { + return "application/octet-stream" + } + return mt +} diff --git a/internal/channels/pancake/message_handler.go b/internal/channels/pancake/message_handler.go new file mode 100644 index 00000000..a17663f7 --- /dev/null +++ b/internal/channels/pancake/message_handler.go @@ -0,0 +1,140 @@ +package pancake + +import ( + "fmt" + "html" + "log/slog" + "strings" + + "github.com/nextlevelbuilder/goclaw/internal/channels" +) + +// handleMessagingEvent converts a Pancake "messaging" webhook event to bus.InboundMessage. +func (ch *Channel) handleMessagingEvent(data MessagingData) { + slog.Debug("pancake: handleMessagingEvent called", + "page_id", ch.pageID, + "sender_id", data.Message.SenderID, + "conversation_id", data.ConversationID, + "type", data.Type, + "platform", data.Platform, + "msg_id", data.Message.ID, + "content_len", len(data.Message.Content)) + + // Dedup by message ID to handle Pancake's at-least-once delivery. + // Skip dedup when message ID is empty — prevents shared slot "msg:" from + // silently dropping all subsequent empty-ID messages across conversations. + var dedupKey string + if data.Message.ID != "" { + dedupKey = fmt.Sprintf("msg:%s", data.Message.ID) + if ch.isDup(dedupKey) { + slog.Info("pancake: duplicate message skipped", "msg_id", data.Message.ID) + return + } + } + + // Prevent reply loops: skip messages sent by the page itself. + if data.Message.SenderID == ch.pageID { + slog.Info("pancake: skipping own page message", + "page_id", ch.pageID, + "sender_id", data.Message.SenderID) + return + } + if isAssignedStaff(data.AssigneeIDs, data.Message.SenderID) { + slog.Info("pancake: skipping assigned staff message", + "page_id", ch.pageID, + "sender_id", data.Message.SenderID, + "conversation_id", data.ConversationID) + return + } + + if data.Message.SenderID == "" { + slog.Warn("pancake: message missing sender_id, skipping", "msg_id", data.Message.ID) + return + } + + // Check echo BEFORE buildMessageContent adds the [From: ...] prefix. + // rememberOutboundEcho stores the raw outbound text; the prefix would cause a + // key mismatch and silently break loop detection. + if ch.isRecentOutboundEcho(data.ConversationID, data.Message.Content) { + slog.Info("pancake: skipping recent outbound echo", + "page_id", ch.pageID, + "conversation_id", data.ConversationID, + "msg_id", data.Message.ID) + return + } + + content := buildMessageContent(data) + + metadata := map[string]string{ + "pancake_mode": strings.ToLower(data.Type), // "inbox" or "comment" + "conversation_type": data.Type, + "platform": data.Platform, + "conversation_id": data.ConversationID, + "message_id": dedupKey, + "display_name": channels.SanitizeDisplayName(data.Message.SenderName), + "page_name": ch.pageName, + } + + ch.HandleMessage( + data.Message.SenderID, + data.ConversationID, // ChatID = conversation_id for reply routing + content, + nil, // media handled inline via content URLs + metadata, + "direct", // Pancake inbox conversations are always treated as direct messages + ) + + slog.Debug("pancake: inbound message published to bus", + "page_id", ch.pageID, + "conv_id", data.ConversationID, + "sender_id", data.Message.SenderID, + "platform", data.Platform, + "type", data.Type, + "channel_name", ch.Name(), + ) +} + +// buildMessageContent combines text content and attachment URLs into a single string. +// Format: [From: {SenderID} ({SenderName})] {content} +func buildMessageContent(data MessagingData) string { + parts := []string{} + + if data.Message.Content != "" { + parts = append(parts, stripHTML(data.Message.Content)) + } + + for _, att := range data.Message.Attachments { + if att.URL != "" { + parts = append(parts, att.URL) + } + } + + body := strings.Join(parts, "\n") + + if data.Message.SenderID != "" { + prefix := fmt.Sprintf("[From: %s (%s)]", data.Message.SenderID, data.Message.SenderName) + if body != "" { + return prefix + " " + body + } + return prefix + } + return body +} + +// stripHTML removes HTML tags and unescapes HTML entities from s. +func stripHTML(s string) string { + s = htmlTagRe.ReplaceAllString(s, "") + return html.UnescapeString(strings.TrimSpace(s)) +} + +func isAssignedStaff(assigneeIDs []string, senderID string) bool { + if senderID == "" { + return false + } + for _, assigneeID := range assigneeIDs { + if assigneeID == senderID { + return true + } + } + return false +} diff --git a/internal/channels/pancake/pancake.go b/internal/channels/pancake/pancake.go new file mode 100644 index 00000000..5ca9e796 --- /dev/null +++ b/internal/channels/pancake/pancake.go @@ -0,0 +1,258 @@ +package pancake + +import ( + "context" + "encoding/json" + "fmt" + "log/slog" + "net/http" + "sync" + "time" + + "github.com/nextlevelbuilder/goclaw/internal/bus" + "github.com/nextlevelbuilder/goclaw/internal/channels" + "github.com/nextlevelbuilder/goclaw/internal/store" +) + +// Compile-time interface assertions. +var ( + _ channels.Channel = (*Channel)(nil) + _ channels.WebhookChannel = (*Channel)(nil) + _ channels.BlockReplyChannel = (*Channel)(nil) +) + +const ( + dedupTTL = 24 * time.Hour + dedupCleanEvery = 5 * time.Minute + outboundEchoTTL = 45 * time.Second +) + +// Channel implements channels.Channel and channels.WebhookChannel for Pancake (pages.fm). +// One channel instance = one Pancake page, which may serve multiple platforms (FB, Zalo, IG, etc.) +type Channel struct { + *channels.BaseChannel + config pancakeInstanceConfig + apiClient *APIClient + pageID string + pageName string // resolved from Pancake page metadata at Start() + platform string // resolved from Pancake page metadata at Start() + webhookSecret string // optional HMAC-SHA256 secret for webhook verification + + // dedup prevents processing duplicate webhook deliveries. + dedup sync.Map // eventKey(string) → time.Time + + // recentOutbound suppresses short-lived webhook echoes of our own text replies. + recentOutbound sync.Map // conversationID + "\x00" + normalized content → time.Time + + stopCh chan struct{} + stopCtx context.Context + stopFn context.CancelFunc +} + +// New creates a Pancake Channel from parsed credentials and config. +func New(cfg pancakeInstanceConfig, creds pancakeCreds, + msgBus *bus.MessageBus, _ store.PairingStore) (*Channel, error) { + + if creds.APIKey == "" { + return nil, fmt.Errorf("pancake: api_key is required") + } + if creds.PageAccessToken == "" { + return nil, fmt.Errorf("pancake: page_access_token is required") + } + if cfg.PageID == "" { + return nil, fmt.Errorf("pancake: page_id is required") + } + + base := channels.NewBaseChannel(channels.TypePancake, msgBus, cfg.AllowFrom) + stopCtx, stopFn := context.WithCancel(context.Background()) + + ch := &Channel{ + BaseChannel: base, + config: cfg, + apiClient: NewAPIClient(creds.APIKey, creds.PageAccessToken, cfg.PageID), + pageID: cfg.PageID, + platform: cfg.Platform, + webhookSecret: creds.WebhookSecret, + stopCh: make(chan struct{}), + stopCtx: stopCtx, + stopFn: stopFn, + } + + return ch, nil +} + +// Factory creates a Pancake Channel from DB instance data. +// Implements channels.ChannelFactory. +func Factory(name string, creds json.RawMessage, cfg json.RawMessage, + msgBus *bus.MessageBus, pairingSvc store.PairingStore) (channels.Channel, error) { + + var c pancakeCreds + if err := json.Unmarshal(creds, &c); err != nil { + return nil, fmt.Errorf("pancake: decode credentials: %w", err) + } + + var ic pancakeInstanceConfig + if len(cfg) > 0 { + if err := json.Unmarshal(cfg, &ic); err != nil { + return nil, fmt.Errorf("pancake: decode config: %w", err) + } + } + + ch, err := New(ic, c, msgBus, pairingSvc) + if err != nil { + return nil, err + } + ch.SetName(name) + return ch, nil +} + +// Start connects the channel: verifies token, resolves platform, registers webhook. +func (ch *Channel) Start(ctx context.Context) error { + ch.MarkStarting("connecting to Pancake page") + + if err := ch.apiClient.VerifyToken(ctx); err != nil { + ch.MarkFailed("token invalid", err.Error(), channels.ChannelFailureKindAuth, false) + return err + } + + // Resolve platform and page name from page metadata (best-effort — don't fail on this). + if ch.platform == "" || ch.pageName == "" { + if page, err := ch.apiClient.GetPage(ctx); err != nil { + slog.Warn("pancake: could not resolve platform from page metadata", "page_id", ch.pageID, "err", err) + } else { + if page.Platform != "" { + ch.platform = page.Platform + } + if page.Name != "" { + ch.pageName = page.Name + } + } + } + + if ch.webhookSecret == "" { + slog.Warn("security.pancake_webhook_no_secret", + "page_id", ch.pageID, + "note", "webhook_secret not configured; incoming webhook requests will not be authenticated") + } + + globalRouter.register(ch) + ch.MarkHealthy("connected to page " + ch.pageID) + ch.SetRunning(true) + + // Background goroutine: evict stale dedup entries to prevent memory growth. + go ch.runDedupCleaner() + + slog.Info("pancake channel started", + "page_id", ch.pageID, + "platform", ch.platform, + "name", ch.Name()) + return nil +} + +// Stop gracefully shuts down the channel. +func (ch *Channel) Stop(_ context.Context) error { + globalRouter.unregister(ch.pageID) + ch.stopFn() + close(ch.stopCh) + ch.SetRunning(false) + ch.MarkStopped("stopped") + slog.Info("pancake channel stopped", "page_id", ch.pageID, "name", ch.Name()) + return nil +} + +// Send delivers an outbound message via Pancake API. +func (ch *Channel) Send(ctx context.Context, msg bus.OutboundMessage) error { + slog.Debug("pancake: Send called", + "page_id", ch.pageID, + "chat_id", msg.ChatID, + "content_len", len(msg.Content), + "platform", ch.platform, + "channel_name", ch.Name()) + + conversationID := msg.ChatID + if conversationID == "" { + return fmt.Errorf("pancake: chat_id (conversation_id) is required for outbound message") + } + + text := FormatOutbound(msg.Content, ch.platform) + + // Handle media attachments. + attachmentIDs, err := ch.handleMediaAttachments(ctx, msg) + if err != nil { + slog.Warn("pancake: media upload failed, sending text only", + "page_id", ch.pageID, "err", err) + } + + // Pancake's official contract makes `message` and `content_ids` mutually exclusive. + // Deliver uploaded media first, then follow with text chunks if needed. + if len(attachmentIDs) > 0 { + if err := ch.apiClient.SendAttachmentMessage(ctx, conversationID, attachmentIDs); err != nil { + ch.handleAPIError(err) + return err + } + if text == "" { + return nil + } + } + + // Text-only: split into platform-appropriate chunks. + // Store echo fingerprints BEFORE sending so that webhook echoes arriving + // while the HTTP round-trip is in flight are already recognized as self-sent. + parts := splitMessage(text, ch.maxMessageLength()) + for _, part := range parts { + ch.rememberOutboundEcho(conversationID, part) + } + for _, part := range parts { + if err := ch.apiClient.SendMessage(ctx, conversationID, part); err != nil { + ch.handleAPIError(err) + // Remove pre-stored echo fingerprints for unsent parts so they + // don't suppress genuine customer messages that happen to match. + ch.forgetOutboundEcho(conversationID, part) + return err + } + } + return nil +} + +// BlockReplyEnabled returns the per-channel block_reply override (nil = inherit gateway default). +func (ch *Channel) BlockReplyEnabled() *bool { return ch.config.BlockReply } + +// WebhookHandler returns the shared webhook path and global router as handler. +// Only the first pancake instance mounts the route; others return ("", nil). +func (ch *Channel) WebhookHandler() (string, http.Handler) { + return globalRouter.webhookRoute() +} + +// handleAPIError maps Pancake API errors to channel health states. +func (ch *Channel) handleAPIError(err error) { + if err == nil { + return + } + switch { + case isAuthError(err): + ch.MarkFailed("token expired or invalid", err.Error(), channels.ChannelFailureKindAuth, false) + case isRateLimitError(err): + ch.MarkDegraded("rate limited", err.Error(), channels.ChannelFailureKindNetwork, true) + default: + ch.MarkDegraded("api error", err.Error(), channels.ChannelFailureKindUnknown, true) + } +} + +// maxMessageLength returns the platform-specific character limit. +func (ch *Channel) maxMessageLength() int { + switch ch.platform { + case "tiktok": + return 500 + case "instagram": + return 1000 + case "facebook", "zalo": + return 2000 + case "whatsapp": + return 4096 + case "line": + return 5000 + default: + return 2000 + } +} + diff --git a/internal/channels/pancake/pancake_loop_regression_test.go b/internal/channels/pancake/pancake_loop_regression_test.go new file mode 100644 index 00000000..4a9e579a --- /dev/null +++ b/internal/channels/pancake/pancake_loop_regression_test.go @@ -0,0 +1,396 @@ +package pancake + +import ( + "context" + "io" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "github.com/nextlevelbuilder/goclaw/internal/bus" + "github.com/nextlevelbuilder/goclaw/internal/channels" +) + +func TestMessageHandlerSkipsRecentOutboundEchoWithHTMLFormatting(t *testing.T) { + msgBus := bus.New() + ch := &Channel{ + BaseChannel: channels.NewBaseChannel(channels.TypePancake, msgBus, nil), + pageID: "page-123", + } + ch.rememberOutboundEcho("conv-1", "Line 1\nLine 2") + + ch.handleMessagingEvent(MessagingData{ + PageID: "page-123", + ConversationID: "conv-1", + Type: "INBOX", + Platform: "facebook", + Message: MessagingMessage{ + ID: "msg-echo-html-1", + SenderID: "user-1", + Content: "Line 1Line 2", + }, + }) + + ctx, cancel := context.WithTimeout(context.Background(), 50*time.Millisecond) + defer cancel() + + if _, ok := msgBus.ConsumeInbound(ctx); ok { + t.Fatal("expected HTML-formatted echo to be dropped") + } +} + +func TestWebhookRouterSkipsNonInboxConversationEvents(t *testing.T) { + msgBus := bus.New() + target := &Channel{ + BaseChannel: channels.NewBaseChannel(channels.TypePancake, msgBus, nil), + pageID: "page-123", + platform: "facebook", + } + router := &webhookRouter{ + instances: map[string]*Channel{ + "page-123": target, + }, + } + + body := `{ + "page_id": "page-123", + "event_type": "messaging", + "data": { + "conversation": { + "id": "page-123_user-1", + "type": "COMMENT", + "from": { + "id": "user-1", + "name": "Customer" + } + }, + "message": { + "id": "msg-comment-1", + "message": "hi" + } + } + }` + + req := httptest.NewRequest(http.MethodPost, "/channels/pancake/webhook", strings.NewReader(body)) + w := httptest.NewRecorder() + router.ServeHTTP(w, req) + + if w.Code != http.StatusOK { + t.Fatalf("status = %d, want %d", w.Code, http.StatusOK) + } + + ctx, cancel := context.WithTimeout(context.Background(), 50*time.Millisecond) + defer cancel() + + if _, ok := msgBus.ConsumeInbound(ctx); ok { + t.Fatal("expected COMMENT conversation webhook to be ignored") + } +} + +func TestWebhookRouterPrefersMessageSenderOverConversationSender(t *testing.T) { + msgBus := bus.New() + target := &Channel{ + BaseChannel: channels.NewBaseChannel(channels.TypePancake, msgBus, nil), + pageID: "page-123", + platform: "facebook", + } + router := &webhookRouter{ + instances: map[string]*Channel{ + "page-123": target, + }, + } + + body := `{ + "page_id": "page-123", + "event_type": "messaging", + "data": { + "conversation": { + "id": "conv-1", + "type": "INBOX", + "from": { + "id": "user-initiator", + "name": "Conversation Starter" + } + }, + "message": { + "id": "msg-actual-sender-1", + "message": "xin chao", + "from": { + "id": "user-actual", + "name": "Actual Sender", + "page_customer_id": "pc-123" + } + } + } + }` + + req := httptest.NewRequest(http.MethodPost, "/channels/pancake/webhook", strings.NewReader(body)) + w := httptest.NewRecorder() + router.ServeHTTP(w, req) + + if w.Code != http.StatusOK { + t.Fatalf("status = %d, want %d", w.Code, http.StatusOK) + } + + ctx, cancel := context.WithTimeout(context.Background(), 50*time.Millisecond) + defer cancel() + + msg, ok := msgBus.ConsumeInbound(ctx) + if !ok { + t.Fatal("expected inbound message to be published") + } + if got, want := msg.SenderID, "user-actual"; got != want { + t.Fatalf("sender_id = %q, want %q", got, want) + } + if got, want := msg.Metadata["display_name"], "Actual Sender"; got != want { + t.Fatalf("metadata.display_name = %q, want %q", got, want) + } +} + +func TestWebhookRouterSkipsPageAuthoredInboxReply(t *testing.T) { + msgBus := bus.New() + target := &Channel{ + BaseChannel: channels.NewBaseChannel(channels.TypePancake, msgBus, nil), + pageID: "page-123", + platform: "facebook", + } + router := &webhookRouter{ + instances: map[string]*Channel{ + "page-123": target, + }, + } + + body := `{ + "page_id": "page-123", + "event_type": "messaging", + "data": { + "conversation": { + "id": "conv-1", + "type": "INBOX", + "from": { + "id": "user-1", + "name": "Customer" + } + }, + "message": { + "id": "msg-page-reply-1", + "message": "manual page reply", + "from": { + "id": "page-123", + "name": "Page Bot" + } + } + } + }` + + req := httptest.NewRequest(http.MethodPost, "/channels/pancake/webhook", strings.NewReader(body)) + w := httptest.NewRecorder() + router.ServeHTTP(w, req) + + if w.Code != http.StatusOK { + t.Fatalf("status = %d, want %d", w.Code, http.StatusOK) + } + + ctx, cancel := context.WithTimeout(context.Background(), 50*time.Millisecond) + defer cancel() + + if _, ok := msgBus.ConsumeInbound(ctx); ok { + t.Fatal("expected page-authored reply not to be published inbound") + } +} + +func TestWebhookRouterSkipsAssignedStaffReply(t *testing.T) { + msgBus := bus.New() + target := &Channel{ + BaseChannel: channels.NewBaseChannel(channels.TypePancake, msgBus, nil), + pageID: "page-123", + platform: "facebook", + } + router := &webhookRouter{ + instances: map[string]*Channel{ + "page-123": target, + }, + } + + body := `{ + "page_id": "page-123", + "event_type": "messaging", + "data": { + "conversation": { + "id": "conv-1", + "type": "INBOX", + "assignee_ids": ["staff-1"], + "from": { + "id": "user-1", + "name": "Customer" + } + }, + "message": { + "id": "msg-staff-reply-1", + "message": "manual staff reply", + "from": { + "id": "staff-1", + "name": "Assigned Staff" + } + } + } + }` + + req := httptest.NewRequest(http.MethodPost, "/channels/pancake/webhook", strings.NewReader(body)) + w := httptest.NewRecorder() + router.ServeHTTP(w, req) + + if w.Code != http.StatusOK { + t.Fatalf("status = %d, want %d", w.Code, http.StatusOK) + } + + ctx, cancel := context.WithTimeout(context.Background(), 50*time.Millisecond) + defer cancel() + + if _, ok := msgBus.ConsumeInbound(ctx); ok { + t.Fatal("expected assigned staff reply not to be published inbound") + } +} + +func TestSendThenWebhookEchoDoesNotRepublishInbound(t *testing.T) { + msgBus := bus.New() + api := NewAPIClient("user-token", "page-token", "page-123") + sendServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + _, _ = io.WriteString(w, `{"success":true}`) + })) + defer sendServer.Close() + + api.pageV1BaseURL = sendServer.URL + api.httpClient = sendServer.Client() + + ch := &Channel{ + BaseChannel: channels.NewBaseChannel(channels.TypePancake, msgBus, nil), + apiClient: api, + pageID: "page-123", + platform: "facebook", + } + + if err := ch.Send(context.Background(), bus.OutboundMessage{ + Channel: "pancake", + ChatID: "conv-1", + Content: "Line 1\nLine 2", + }); err != nil { + t.Fatalf("Send returned error: %v", err) + } + + router := &webhookRouter{ + instances: map[string]*Channel{ + "page-123": ch, + }, + } + + body := `{ + "page_id": "page-123", + "event_type": "messaging", + "data": { + "conversation": { + "id": "conv-1", + "type": "INBOX", + "from": { + "id": "user-1", + "name": "Customer" + } + }, + "message": { + "id": "msg-echo-roundtrip-1", + "message": "Line 1Line 2" + } + } + }` + + req := httptest.NewRequest(http.MethodPost, "/channels/pancake/webhook", strings.NewReader(body)) + w := httptest.NewRecorder() + router.ServeHTTP(w, req) + + if w.Code != http.StatusOK { + t.Fatalf("status = %d, want %d", w.Code, http.StatusOK) + } + + ctx, cancel := context.WithTimeout(context.Background(), 50*time.Millisecond) + defer cancel() + + if _, ok := msgBus.ConsumeInbound(ctx); ok { + t.Fatal("expected echoed outbound send not to republish inbound work") + } +} + +// TestSendRaceConditionEchoArrivesBeforeSendReturns simulates the production race: +// webhook echo arriving while SendMessage HTTP call is still in flight. +// Before the fix, rememberOutboundEcho was called AFTER SendMessage, so an echo +// arriving during the HTTP round-trip would not be recognized. +func TestSendRaceConditionEchoArrivesBeforeSendReturns(t *testing.T) { + msgBus := bus.New() + api := NewAPIClient("user-token", "page-token", "page-123") + + ch := &Channel{ + BaseChannel: channels.NewBaseChannel(channels.TypePancake, msgBus, nil), + apiClient: api, + pageID: "page-123", + platform: "facebook", + } + + router := &webhookRouter{ + instances: map[string]*Channel{ + "page-123": ch, + }, + } + + const outboundContent = "Em đây, alive luôn 😊\nThoát loop thành công rồi nha." + + // Pancake API server that delivers the echo webhook BEFORE returning the + // SendMessage response — simulating the real-world race condition. + sendServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // While SendMessage is "in flight", deliver the echo webhook. + echoBody := `{ + "page_id": "page-123", + "event_type": "messaging", + "data": { + "conversation": { + "id": "conv-race", + "type": "INBOX", + "from": {"id": "user-1", "name": "Customer"} + }, + "message": { + "id": "msg-race-echo", + "message": "Em đây, alive luôn 😊 Thoát loop thành công rồi nha." + } + } + }` + echoReq := httptest.NewRequest(http.MethodPost, "/channels/pancake/webhook", + strings.NewReader(echoBody)) + echoW := httptest.NewRecorder() + router.ServeHTTP(echoW, echoReq) + + // Now return SendMessage success. + w.Header().Set("Content-Type", "application/json") + _, _ = io.WriteString(w, `{"success":true}`) + })) + defer sendServer.Close() + + api.pageV1BaseURL = sendServer.URL + api.httpClient = sendServer.Client() + + if err := ch.Send(context.Background(), bus.OutboundMessage{ + Channel: "pancake", + ChatID: "conv-race", + Content: outboundContent, + }); err != nil { + t.Fatalf("Send returned error: %v", err) + } + + // The echo webhook was delivered during Send. It must NOT have been published. + ctx, cancel := context.WithTimeout(context.Background(), 50*time.Millisecond) + defer cancel() + + if _, ok := msgBus.ConsumeInbound(ctx); ok { + t.Fatal("echo arrived during SendMessage HTTP call but was not suppressed — race condition not fixed") + } +} diff --git a/internal/channels/pancake/pancake_test.go b/internal/channels/pancake/pancake_test.go new file mode 100644 index 00000000..c1506c36 --- /dev/null +++ b/internal/channels/pancake/pancake_test.go @@ -0,0 +1,484 @@ +package pancake + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + "unicode/utf8" + + "github.com/nextlevelbuilder/goclaw/internal/bus" + "github.com/nextlevelbuilder/goclaw/internal/channels" +) + +// TestFactory_Valid verifies Factory creates a Channel from valid JSON creds/config. +func TestFactory_Valid(t *testing.T) { + creds, _ := json.Marshal(pancakeCreds{ + APIKey: "test-api-key", + PageAccessToken: "test-page-token", + }) + cfg, _ := json.Marshal(pancakeInstanceConfig{PageID: "12345"}) + + ch, err := Factory("pancake-test", creds, cfg, nil, nil) + if err != nil { + t.Fatalf("Factory returned unexpected error: %v", err) + } + if ch == nil { + t.Fatal("Factory returned nil channel") + } + if ch.Name() != "pancake-test" { + t.Errorf("Name() = %q, want %q", ch.Name(), "pancake-test") + } +} + +// TestFactory_MissingAPIKey verifies Factory returns error when api_key is empty. +func TestFactory_MissingAPIKey(t *testing.T) { + creds, _ := json.Marshal(pancakeCreds{PageAccessToken: "token"}) + cfg, _ := json.Marshal(pancakeInstanceConfig{PageID: "12345"}) + + _, err := Factory("test", creds, cfg, nil, nil) + if err == nil { + t.Fatal("expected error for missing api_key, got nil") + } +} + +// TestFactory_MissingPageAccessToken verifies Factory returns error when page_access_token is empty. +func TestFactory_MissingPageAccessToken(t *testing.T) { + creds, _ := json.Marshal(pancakeCreds{APIKey: "key"}) + cfg, _ := json.Marshal(pancakeInstanceConfig{PageID: "12345"}) + + _, err := Factory("test", creds, cfg, nil, nil) + if err == nil { + t.Fatal("expected error for missing page_access_token, got nil") + } +} + +// TestFactory_MissingPageID verifies Factory returns error when page_id is empty. +func TestFactory_MissingPageID(t *testing.T) { + creds, _ := json.Marshal(pancakeCreds{ + APIKey: "key", + PageAccessToken: "token", + }) + cfg, _ := json.Marshal(pancakeInstanceConfig{}) // no page_id + + _, err := Factory("test", creds, cfg, nil, nil) + if err == nil { + t.Fatal("expected error for missing page_id, got nil") + } +} + +// TestFormatOutbound verifies platform-aware formatting for each platform. +func TestFormatOutbound(t *testing.T) { + input := "**Hello** _world_ `code` ## Header [link](http://example.com)" + + cases := []struct { + platform string + wantNot string // substring that should NOT appear in output + }{ + {"facebook", "**"}, + {"zalo", "**"}, + {"instagram", "_"}, + {"tiktok", "##"}, + {"whatsapp", "**"}, + {"line", "##"}, + {"unknown", "`"}, + } + + for _, tc := range cases { + t.Run(tc.platform, func(t *testing.T) { + out := FormatOutbound(input, tc.platform) + if out == "" { + t.Error("FormatOutbound returned empty string") + } + _ = out // formatting verified visually; we just check no panic + non-empty + }) + } +} + +// TestSplitMessage verifies message splitting at platform character limits. +func TestSplitMessage(t *testing.T) { + t.Run("short message not split", func(t *testing.T) { + parts := splitMessage("hello", 100) + if len(parts) != 1 || parts[0] != "hello" { + t.Errorf("unexpected parts: %v", parts) + } + }) + + t.Run("exact limit not split", func(t *testing.T) { + msg := string(make([]byte, 100)) + parts := splitMessage(msg, 100) + if len(parts) != 1 { + t.Errorf("expected 1 part, got %d", len(parts)) + } + }) + + t.Run("over limit is split", func(t *testing.T) { + msg := string(make([]byte, 250)) + parts := splitMessage(msg, 100) + if len(parts) != 3 { + t.Errorf("expected 3 parts, got %d", len(parts)) + } + }) + + t.Run("zero limit returns whole string", func(t *testing.T) { + parts := splitMessage("hello", 0) + if len(parts) != 1 { + t.Errorf("expected 1 part with zero limit, got %d", len(parts)) + } + }) +} + +// TestIsDup verifies dedup returns false first, true on repeat. +func TestIsDup(t *testing.T) { + ch := &Channel{} + + if ch.isDup("key-1") { + t.Error("isDup: first call should return false") + } + if !ch.isDup("key-1") { + t.Error("isDup: second call should return true") + } + if ch.isDup("key-2") { + t.Error("isDup: different key should return false") + } +} + +// TestWebhookRouterReturns200 verifies the global router always returns HTTP 200. +func TestWebhookRouterReturns200(t *testing.T) { + // Use a fresh local router to avoid interfering with the package-level globalRouter. + router := &webhookRouter{instances: make(map[string]*Channel)} + + t.Run("POST event returns 200", func(t *testing.T) { + body := `{"data":{"conversation":{"id":"123_456","type":"INBOX","from":{"id":"456"}},"message":{"id":"m1"}}}` + req := httptest.NewRequest(http.MethodPost, "/channels/pancake/webhook", + strings.NewReader(body)) + w := httptest.NewRecorder() + router.ServeHTTP(w, req) + if w.Code != http.StatusOK { + t.Errorf("expected 200, got %d", w.Code) + } + }) + + t.Run("GET returns 200 (not 405)", func(t *testing.T) { + req := httptest.NewRequest(http.MethodGet, "/channels/pancake/webhook", nil) + w := httptest.NewRecorder() + router.ServeHTTP(w, req) + if w.Code != http.StatusOK { + t.Errorf("expected 200, got %d", w.Code) + } + }) + + t.Run("malformed JSON returns 200", func(t *testing.T) { + req := httptest.NewRequest(http.MethodPost, "/channels/pancake/webhook", + strings.NewReader("not-json")) + w := httptest.NewRecorder() + router.ServeHTTP(w, req) + if w.Code != http.StatusOK { + t.Errorf("expected 200, got %d", w.Code) + } + }) +} + +// TestMessageHandlerSkipsSelfReply verifies the page's own messages are not published. +// The dedup entry is stored (dedup runs first), but HandleMessage is never called. +// If the self-reply guard were absent, ch.bus (nil) would panic — making no-panic the assertion. +func TestMessageHandlerSkipsSelfReply(t *testing.T) { + const pageID = "page-123" + ch := &Channel{pageID: pageID} + + data := MessagingData{ + PageID: pageID, + ConversationID: "conv-1", + Type: "INBOX", + Platform: "facebook", + Message: MessagingMessage{ + ID: "msg-self-1", + SenderID: pageID, // same as page → must be skipped before HandleMessage + SenderName: "Page Bot", + Content: "Hello", + }, + } + + // Must not panic. If self-reply guard is missing, nil bus dereference panics here. + ch.handleMessagingEvent(data) + + // Dedup entry is stored (dedup check runs before self-reply check). + _, stored := ch.dedup.Load("msg:msg-self-1") + if !stored { + t.Error("dedup entry should have been stored (dedup runs before self-reply guard)") + } +} + +func TestMessageHandlerPublishesMessageIDMetadata(t *testing.T) { + msgBus := bus.New() + ch := &Channel{ + BaseChannel: channels.NewBaseChannel(channels.TypePancake, msgBus, nil), + pageID: "page-123", + } + + ch.handleMessagingEvent(MessagingData{ + PageID: "page-123", + ConversationID: "conv-1", + Type: "INBOX", + Platform: "facebook", + Message: MessagingMessage{ + ID: "msg-123", + SenderID: "user-1", + Content: "hello", + }, + }) + + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + + msg, ok := msgBus.ConsumeInbound(ctx) + if !ok { + t.Fatal("expected inbound message to be published") + } + if got, want := msg.Metadata["message_id"], "msg:msg-123"; got != want { + t.Fatalf("metadata.message_id = %q, want %q", got, want) + } +} + +func TestMessageHandlerSkipsRecentOutboundEcho(t *testing.T) { + msgBus := bus.New() + ch := &Channel{ + BaseChannel: channels.NewBaseChannel(channels.TypePancake, msgBus, nil), + pageID: "page-123", + } + ch.rememberOutboundEcho("conv-1", "hello from bot") + + ch.handleMessagingEvent(MessagingData{ + PageID: "page-123", + ConversationID: "conv-1", + Type: "INBOX", + Platform: "facebook", + Message: MessagingMessage{ + ID: "msg-echo-1", + SenderID: "user-1", + Content: "hello from bot", + }, + }) + + ctx, cancel := context.WithTimeout(context.Background(), 50*time.Millisecond) + defer cancel() + + if _, ok := msgBus.ConsumeInbound(ctx); ok { + t.Fatal("expected echoed outbound message to be dropped") + } +} + +func TestBlockReplyEnabledUsesChannelOverride(t *testing.T) { + enabled := true + ch := &Channel{ + config: pancakeInstanceConfig{ + BlockReply: &enabled, + }, + } + + got := ch.BlockReplyEnabled() + if got == nil || !*got { + t.Fatalf("BlockReplyEnabled() = %v, want true", got) + } +} + +type captureTransport struct { + req *http.Request + body []byte + resp *http.Response +} + +func (t *captureTransport) RoundTrip(req *http.Request) (*http.Response, error) { + t.req = req.Clone(req.Context()) + if req.Body != nil { + body, err := io.ReadAll(req.Body) + if err != nil { + return nil, err + } + t.body = body + req.Body = io.NopCloser(bytes.NewReader(body)) + } + if t.resp != nil { + return t.resp, nil + } + return &http.Response{ + StatusCode: http.StatusOK, + Header: make(http.Header), + Body: io.NopCloser(strings.NewReader(`{"success":true}`)), + Request: req, + }, nil +} + +func TestAPIClientSendMessageMatchesOfficialContract(t *testing.T) { + transport := &captureTransport{} + client := NewAPIClient("user-token", "page-token", "page-123") + client.httpClient = &http.Client{Transport: transport} + + if err := client.SendMessage(context.Background(), "conv-456", "xin chao"); err != nil { + t.Fatalf("SendMessage returned error: %v", err) + } + + if transport.req == nil { + t.Fatal("expected outbound request to be captured") + } + if got, want := transport.req.URL.Path, "/api/public_api/v1/pages/page-123/conversations/conv-456/messages"; got != want { + t.Fatalf("request path = %q, want %q", got, want) + } + if got := transport.req.URL.Query().Get("page_access_token"); got != "page-token" { + t.Fatalf("page_access_token query = %q, want %q", got, "page-token") + } + + var payload map[string]any + if err := json.Unmarshal(transport.body, &payload); err != nil { + t.Fatalf("unmarshal payload: %v", err) + } + if got, want := payload["action"], "reply_inbox"; got != want { + t.Fatalf("payload.action = %#v, want %#v", got, want) + } + if got, want := payload["message"], "xin chao"; got != want { + t.Fatalf("payload.message = %#v, want %#v", got, want) + } + if _, exists := payload["content"]; exists { + t.Fatalf("payload must not contain legacy content field: %s", string(transport.body)) + } + if _, exists := payload["attachment_id"]; exists { + t.Fatalf("payload must not contain attachment_id field: %s", string(transport.body)) + } +} + +func TestAPIClientSendMessageReturnsBodyLevelError(t *testing.T) { + transport := &captureTransport{ + resp: &http.Response{ + StatusCode: http.StatusOK, + Header: make(http.Header), + Body: io.NopCloser(strings.NewReader(`{"success":false,"message":"conversation blocked"}`)), + }, + } + client := NewAPIClient("user-token", "page-token", "page-123") + client.httpClient = &http.Client{Transport: transport} + + err := client.SendMessage(context.Background(), "conv-456", "xin chao") + if err == nil { + t.Fatal("expected SendMessage to return body-level error") + } + if !strings.Contains(err.Error(), "conversation blocked") { + t.Fatalf("SendMessage error = %v, want body-level message", err) + } +} + +// TestTruncateForTikTok_MultiByteCharacters verifies rune-safe truncation for +// Vietnamese diacritics and emoji (multi-byte UTF-8 sequences). +func TestTruncateForTikTok_MultiByteCharacters(t *testing.T) { + // Vietnamese text with diacritics (multi-byte UTF-8) + input := strings.Repeat("Xin chào ", 100) // ~900 bytes, <500 runes + result := truncateForTikTok(input) + if !utf8.ValidString(result) { + t.Fatal("truncateForTikTok produced invalid UTF-8") + } + + // Emoji string exceeding 500 runes + emoji := strings.Repeat("😊", 600) + result = truncateForTikTok(emoji) + runes := []rune(result) + if len(runes) > 500 { + t.Errorf("expected <=500 runes, got %d", len(runes)) + } + if !utf8.ValidString(result) { + t.Fatal("emoji truncation produced invalid UTF-8") + } +} + +// TestMessageHandlerEmptyMessageID verifies that two messages with empty IDs +// from different conversations are both published (not deduped against each other). +func TestMessageHandlerEmptyMessageID(t *testing.T) { + msgBus := bus.New() + ch := &Channel{ + BaseChannel: channels.NewBaseChannel(channels.TypePancake, msgBus, nil), + pageID: "page-123", + } + + // First message with empty ID — should be published + ch.handleMessagingEvent(MessagingData{ + PageID: "page-123", ConversationID: "conv-1", + Type: "INBOX", Platform: "facebook", + Message: MessagingMessage{ID: "", SenderID: "user-1", Content: "hello"}, + }) + + // Second message with empty ID, different conversation — should ALSO be published + ch.handleMessagingEvent(MessagingData{ + PageID: "page-123", ConversationID: "conv-2", + Type: "INBOX", Platform: "facebook", + Message: MessagingMessage{ID: "", SenderID: "user-2", Content: "world"}, + }) + + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + + _, ok1 := msgBus.ConsumeInbound(ctx) + if !ok1 { + t.Fatal("first empty-ID message should be published") + } + _, ok2 := msgBus.ConsumeInbound(ctx) + if !ok2 { + t.Fatal("second empty-ID message should NOT be deduped against first") + } +} + +func TestAPIClientUploadMediaMatchesOfficialContract(t *testing.T) { + transport := &captureTransport{ + resp: &http.Response{ + StatusCode: http.StatusOK, + Header: make(http.Header), + Body: io.NopCloser(strings.NewReader(`{"id":"upload-123","success":true}`)), + }, + } + client := NewAPIClient("user-token", "page-token", "page-123") + client.httpClient = &http.Client{Transport: transport} + + id, err := client.UploadMedia(context.Background(), "photo.jpg", strings.NewReader("file-bytes"), "image/jpeg") + if err != nil { + t.Fatalf("UploadMedia returned error: %v", err) + } + if id != "upload-123" { + t.Fatalf("UploadMedia id = %q, want %q", id, "upload-123") + } + if transport.req == nil { + t.Fatal("expected upload request to be captured") + } + if got, want := transport.req.URL.Path, "/api/public_api/v1/pages/page-123/upload_contents"; got != want { + t.Fatalf("upload path = %q, want %q", got, want) + } + if got := transport.req.URL.Query().Get("page_access_token"); got != "page-token" { + t.Fatalf("upload page_access_token query = %q, want %q", got, "page-token") + } + if !strings.HasPrefix(transport.req.Header.Get("Content-Type"), "multipart/form-data; boundary=") { + t.Fatalf("upload Content-Type = %q, want multipart/form-data", transport.req.Header.Get("Content-Type")) + } +} + +// TestIsAuthError_WrappedError verifies errors.As works with wrapped apiError. +func TestIsAuthError_WrappedError(t *testing.T) { + inner := &apiError{Code: 401, Message: "unauthorized"} + wrapped := fmt.Errorf("send failed: %w", inner) + if !isAuthError(wrapped) { + t.Error("isAuthError should detect wrapped 401 apiError via errors.As") + } + if isAuthError(fmt.Errorf("random error")) { + t.Error("isAuthError should return false for non-apiError") + } +} + +// TestIsRateLimitError_WrappedError verifies errors.As works with wrapped rate limit. +func TestIsRateLimitError_WrappedError(t *testing.T) { + inner := &apiError{Code: 429, Message: "too many requests"} + wrapped := fmt.Errorf("send failed: %w", inner) + if !isRateLimitError(wrapped) { + t.Error("isRateLimitError should detect wrapped 429 apiError via errors.As") + } +} diff --git a/internal/channels/pancake/types.go b/internal/channels/pancake/types.go new file mode 100644 index 00000000..a0a239df --- /dev/null +++ b/internal/channels/pancake/types.go @@ -0,0 +1,130 @@ +// Package pancake implements the Pancake (pages.fm) channel for GoClaw. +// Pancake acts as a unified proxy for Facebook, Zalo OA, Instagram, TikTok, WhatsApp, Line. +// A single Pancake API key gives access to all connected platforms — no per-platform OAuth needed. +package pancake + +import "encoding/json" + +// pancakeCreds holds encrypted credentials stored in channel_instances.credentials. +type pancakeCreds struct { + APIKey string `json:"api_key"` // User-level Pancake API key + PageAccessToken string `json:"page_access_token"` // Page-level token for all page APIs + WebhookSecret string `json:"webhook_secret,omitempty"` // Optional HMAC-SHA256 verification +} + +// pancakeInstanceConfig holds non-secret config from channel_instances.config JSONB. +type pancakeInstanceConfig struct { + PageID string `json:"page_id"` + Platform string `json:"platform,omitempty"` // auto-detected at Start(): facebook/zalo/instagram/tiktok/whatsapp/line + Features struct { + InboxReply bool `json:"inbox_reply"` + CommentReply bool `json:"comment_reply"` + } `json:"features"` + AllowFrom []string `json:"allow_from,omitempty"` + BlockReply *bool `json:"block_reply,omitempty"` // override gateway block_reply (nil = inherit) +} + +// --- Webhook payload types --- +// These types match the actual Pancake (pages.fm) webhook delivery format. +// Top-level envelope has optional "event_type" + nested "data" containing +// "conversation" and "message" objects. + +// WebhookEvent is the top-level Pancake webhook delivery envelope. +type WebhookEvent struct { + EventType string `json:"event_type,omitempty"` // "messaging", may be empty + PageID string `json:"page_id,omitempty"` // top-level page_id (some formats) + Data json.RawMessage `json:"data"` +} + +// WebhookData is the "data" envelope inside a Pancake webhook event. +type WebhookData struct { + Conversation WebhookConversation `json:"conversation"` + Message WebhookMessage `json:"message"` + PageID string `json:"page_id,omitempty"` // page_id may appear here or top-level +} + +// WebhookConversation holds the conversation metadata from a Pancake webhook. +type WebhookConversation struct { + ID string `json:"id"` // format: "pageID_senderID" + Type string `json:"type"` // "INBOX" or "COMMENT" + AssigneeIDs []string `json:"assignee_ids,omitempty"` // Pancake staff IDs assigned to this conversation + From WebhookSender `json:"from"` + Snippet string `json:"snippet,omitempty"` +} + +// WebhookSender identifies the message sender. +type WebhookSender struct { + ID string `json:"id"` + Name string `json:"name"` + Email string `json:"email,omitempty"` + PageCustomerID string `json:"page_customer_id,omitempty"` +} + +// WebhookMessage holds the message payload from a Pancake webhook. +type WebhookMessage struct { + ID string `json:"id"` + Message string `json:"message,omitempty"` // primary text content + OriginalMessage string `json:"original_message,omitempty"` // unformatted fallback + Content string `json:"content,omitempty"` // legacy field + From *WebhookSender `json:"from,omitempty"` // actual sender of this message + Attachments []MessageAttachment `json:"attachments,omitempty"` + CreatedAt json.Number `json:"created_at,omitempty"` +} + +// MessageAttachment represents a media attachment in a Pancake webhook message. +type MessageAttachment struct { + Type string `json:"type"` // "image", "video", "file" + URL string `json:"url"` +} + +// MessagingData is the normalized internal representation used after parsing. +type MessagingData struct { + PageID string + ConversationID string + Type string // "INBOX" or "COMMENT" + Platform string // "facebook", "zalo", "instagram", "tiktok", "whatsapp", "line" + AssigneeIDs []string + Message MessagingMessage +} + +// MessagingMessage is the normalized message used by the handler. +type MessagingMessage struct { + ID string + Content string + SenderID string + SenderName string + Attachments []MessageAttachment +} + +// --- API response types --- + +// PageInfo holds page metadata from GET /pages response. +type PageInfo struct { + ID string `json:"id"` + Name string `json:"name"` + Platform string `json:"platform"` // facebook/zalo/instagram/tiktok/whatsapp/line + Avatar string `json:"avatar,omitempty"` +} + +// SendMessageRequest is the POST body for sending a message via Pancake API. +type SendMessageRequest struct { + Action string `json:"action"` + Message string `json:"message,omitempty"` + ContentIDs []string `json:"content_ids,omitempty"` +} + +// UploadResponse is returned by POST /pages/{id}/upload_contents. +type UploadResponse struct { + ID string `json:"id"` + URL string `json:"url,omitempty"` +} + +// apiError wraps a Pancake API error response. +type apiError struct { + Code int `json:"code"` + Message string `json:"message"` +} + +func (e *apiError) Error() string { + return e.Message +} diff --git a/internal/channels/pancake/webhook_handler.go b/internal/channels/pancake/webhook_handler.go new file mode 100644 index 00000000..981f0cc5 --- /dev/null +++ b/internal/channels/pancake/webhook_handler.go @@ -0,0 +1,242 @@ +package pancake + +import ( + "crypto/hmac" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "io" + "log/slog" + "net/http" + "strings" + "sync" +) + +const ( + webhookPath = "/channels/pancake/webhook" + maxBodyBytes = 1 << 20 // 1 MB — prevent abuse +) + +// verifyHMAC verifies a Pancake HMAC-SHA256 signature. +// Expected header format: "sha256=" +func verifyHMAC(body []byte, secret, signature string) bool { + const prefix = "sha256=" + if len(signature) <= len(prefix) { + return false + } + got, err := hex.DecodeString(signature[len(prefix):]) + if err != nil { + return false + } + mac := hmac.New(sha256.New, []byte(secret)) + mac.Write(body) + expected := mac.Sum(nil) + return hmac.Equal(got, expected) +} + +// --- Global webhook router for multi-page support --- + +// webhookRouter routes incoming Pancake webhook events to the correct channel instance by page_id. +// A single HTTP handler is shared across all pancake channel instances. +type webhookRouter struct { + mu sync.RWMutex + instances map[string]*Channel // pageID → channel + routeHandled bool // true after first webhookRoute() call +} + +var globalRouter = &webhookRouter{ + instances: make(map[string]*Channel), +} + +func (r *webhookRouter) register(ch *Channel) { + r.mu.Lock() + defer r.mu.Unlock() + r.instances[ch.pageID] = ch +} + +func (r *webhookRouter) unregister(pageID string) { + r.mu.Lock() + defer r.mu.Unlock() + delete(r.instances, pageID) +} + +// webhookRoute returns the path+handler on first call; ("", nil) for subsequent calls. +// The HTTP mux retains the route once registered — routeHandled prevents duplicate mounts. +func (r *webhookRouter) webhookRoute() (string, http.Handler) { + r.mu.Lock() + defer r.mu.Unlock() + if !r.routeHandled { + r.routeHandled = true + return webhookPath, r + } + return "", nil +} + +// ServeHTTP is the shared handler for all Pancake page webhooks. +// Always returns HTTP 200 — Pancake suspends webhooks if >80% errors in a 30-min window. +func (r *webhookRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) { + slog.Info("pancake: webhook request received", + "method", req.Method, + "remote_addr", req.RemoteAddr, + "content_length", req.ContentLength) + + if req.Method != http.MethodPost { + w.WriteHeader(http.StatusOK) + return + } + + lr := io.LimitReader(req.Body, maxBodyBytes+1) + body, err := io.ReadAll(lr) + if err != nil { + slog.Warn("pancake: router read body error", "err", err) + w.WriteHeader(http.StatusOK) + return + } + + slog.Debug("pancake: webhook body received", + "body_len", len(body), + "body_preview", truncateBody(body, 1000)) + + // Parse top-level envelope. + var event WebhookEvent + if err := json.Unmarshal(body, &event); err != nil { + slog.Warn("pancake: router parse event error", "err", err, "body_preview", truncateBody(body, 300)) + w.WriteHeader(http.StatusOK) + return + } + + // Parse the nested "data" object containing conversation + message. + var data WebhookData + if err := json.Unmarshal(event.Data, &data); err != nil { + slog.Warn("pancake: router parse data error", "err", err) + w.WriteHeader(http.StatusOK) + return + } + + // Resolve page_id: try top-level first, then data-level, then extract from conversation ID. + pageID := event.PageID + if pageID == "" { + pageID = data.PageID + } + if pageID == "" { + // Conversation ID format: "pageID_senderID" — extract page portion. + if idx := strings.Index(data.Conversation.ID, "_"); idx > 0 { + pageID = data.Conversation.ID[:idx] + } + } + + // Resolve conversation type — only process INBOX messages. + convType := strings.ToUpper(data.Conversation.Type) + + if event.EventType != "" && !strings.EqualFold(event.EventType, "messaging") { + slog.Debug("pancake: skipping non-messaging webhook event", + "event_type", event.EventType, + "page_id", pageID) + w.WriteHeader(http.StatusOK) + return + } + + slog.Debug("pancake: webhook event parsed", + "event_type", event.EventType, + "page_id", pageID, + "conv_id", data.Conversation.ID, + "conv_type", convType, + "sender_id", data.Conversation.From.ID, + "sender_name", data.Conversation.From.Name, + "msg_id", data.Message.ID) + + if pageID == "" { + slog.Warn("pancake: could not determine page_id from webhook payload") + w.WriteHeader(http.StatusOK) + return + } + if convType != "INBOX" { + slog.Info("pancake: skipping non-inbox conversation event", + "page_id", pageID, + "conv_id", data.Conversation.ID, + "conv_type", convType, + "msg_id", data.Message.ID) + w.WriteHeader(http.StatusOK) + return + } + + r.mu.RLock() + target := r.instances[pageID] + r.mu.RUnlock() + + if target == nil { + // Log all registered page IDs for debugging. + r.mu.RLock() + var registered []string + for pid := range r.instances { + registered = append(registered, pid) + } + r.mu.RUnlock() + slog.Warn("pancake: no channel instance for page_id", + "page_id", pageID, + "registered_pages", registered) + w.WriteHeader(http.StatusOK) + return + } + + // HMAC signature verification — skip if webhook_secret not configured. + if target.webhookSecret != "" { + sig := req.Header.Get("X-Pancake-Signature") + if !verifyHMAC(body, target.webhookSecret, sig) { + slog.Warn("security.pancake_webhook_signature_mismatch", + "page_id", pageID, + "remote_addr", req.RemoteAddr) + w.WriteHeader(http.StatusOK) + return + } + } + + // Build normalized MessagingData from actual Pancake payload. + msgContent := data.Message.Message + if msgContent == "" { + msgContent = data.Message.OriginalMessage + } + if msgContent == "" { + msgContent = data.Message.Content + } + if msgContent == "" { + msgContent = data.Conversation.Snippet + } + + senderID := data.Conversation.From.ID + senderName := data.Conversation.From.Name + if data.Message.From != nil { + if data.Message.From.ID != "" { + senderID = data.Message.From.ID + } + if data.Message.From.Name != "" { + senderName = data.Message.From.Name + } + } + + normalized := MessagingData{ + PageID: pageID, + ConversationID: data.Conversation.ID, + Type: convType, + Platform: target.platform, + AssigneeIDs: append([]string(nil), data.Conversation.AssigneeIDs...), + Message: MessagingMessage{ + ID: data.Message.ID, + Content: msgContent, + SenderID: senderID, + SenderName: senderName, + Attachments: data.Message.Attachments, + }, + } + + target.handleMessagingEvent(normalized) + w.WriteHeader(http.StatusOK) +} + +// truncateBody returns a string preview of body, truncated to maxLen bytes. +func truncateBody(body []byte, maxLen int) string { + if len(body) <= maxLen { + return string(body) + } + return string(body[:maxLen]) + "..." +} diff --git a/internal/channels/sanitize_display_name.go b/internal/channels/sanitize_display_name.go new file mode 100644 index 00000000..f674c84a --- /dev/null +++ b/internal/channels/sanitize_display_name.go @@ -0,0 +1,17 @@ +package channels + +import "strings" + +// SanitizeDisplayName strips newlines and markdown heading markers from a +// channel-provided display name to prevent injection into markdown templates +// (e.g. USER.md). Caps length at 100 runes. +func SanitizeDisplayName(name string) string { + name = strings.ReplaceAll(name, "\n", " ") + name = strings.ReplaceAll(name, "\r", " ") + name = strings.ReplaceAll(name, "#", "") + name = strings.TrimSpace(name) + if len([]rune(name)) > 100 { + name = string([]rune(name)[:100]) + } + return name +} diff --git a/internal/channels/slack/handlers.go b/internal/channels/slack/handlers.go index 510f201d..fa9e2999 100644 --- a/internal/channels/slack/handlers.go +++ b/internal/channels/slack/handlers.go @@ -251,7 +251,7 @@ func (c *Channel) handleMessage(ev *slackevents.MessageEvent) { "message_id": ev.TimeStamp, "user_id": senderID, "username": displayName, - "display_name": displayName, + "display_name": channels.SanitizeDisplayName(displayName), "channel_id": channelID, "is_dm": fmt.Sprintf("%t", isDM), "local_key": localKey, diff --git a/internal/channels/zalo/personal/handlers.go b/internal/channels/zalo/personal/handlers.go index e7999cf2..0546b7b1 100644 --- a/internal/channels/zalo/personal/handlers.go +++ b/internal/channels/zalo/personal/handlers.go @@ -70,7 +70,7 @@ func (c *Channel) handleDM(msg protocol.UserMessage) { metadata := map[string]string{ "message_id": msg.Data.MsgID, "platform": channels.TypeZaloPersonal, - "display_name": senderName, + "display_name": channels.SanitizeDisplayName(senderName), } c.HandleMessage(senderID, threadID, content, media, metadata, "direct") } @@ -151,7 +151,7 @@ func (c *Channel) handleGroupMessage(msg protocol.GroupMessage) { "message_id": msg.Data.MsgID, "platform": channels.TypeZaloPersonal, "group_id": threadID, - "display_name": senderName, + "display_name": channels.SanitizeDisplayName(senderName), } c.HandleMessage(senderID, threadID, finalContent, allMedia, metadata, "group") diff --git a/internal/config/config.go b/internal/config/config.go index e995915d..b1639d41 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -3,6 +3,7 @@ package config import ( "encoding/json" "fmt" + "log/slog" "sync" "time" @@ -186,6 +187,22 @@ type MemoryConfig struct { VectorWeight float64 `json:"vector_weight,omitempty"` // hybrid search vector weight (default 0.7) TextWeight float64 `json:"text_weight,omitempty"` // hybrid search FTS weight (default 0.3) MinScore float64 `json:"min_score,omitempty"` // minimum relevance score (default 0.35) + + // Dreaming configures the episodic → long-term consolidation worker. + // nil = use hardcoded defaults (threshold=5, debounce=10min, enabled). + Dreaming *DreamingConfig `json:"dreaming,omitempty"` +} + +// DreamingConfig controls per-agent behaviour of the consolidation dreaming +// worker (episodic summaries → long-term memory). Pointer fields allow partial +// overrides from JSONB to merge cleanly with defaults without clobbering +// unset values (e.g. leaving VerboseLog at its default when only Threshold is +// overridden). +type DreamingConfig struct { + Enabled *bool `json:"enabled,omitempty"` // default true (nil = enabled) + DebounceMs int `json:"debounce_ms,omitempty"` // min interval between runs per agent/user (default 600000 = 10 min) + Threshold int `json:"threshold,omitempty"` // min unpromoted entries before running (default 5) + VerboseLog *bool `json:"verbose_log,omitempty"` // log debounce/below-threshold skips at info level (default false) } // SandboxConfig configures Docker-based sandbox execution. @@ -301,6 +318,11 @@ type ModelPricing struct { OutputPerMillion float64 `json:"output_per_million"` CacheReadPerMillion float64 `json:"cache_read_per_million,omitempty"` CacheCreatePerMillion float64 `json:"cache_create_per_million,omitempty"` + // ReasoningPerMillion is the per-million-token rate for reasoning/thinking tokens + // (e.g., Claude extended thinking, GPT-5 reasoning, o3/o4-mini CoT). If zero, + // reasoning tokens fall back to OutputPerMillion (providers typically charge + // reasoning at the same rate as output tokens). + ReasoningPerMillion float64 `json:"reasoning_per_million,omitempty"` } // TelemetryConfig configures OpenTelemetry export for traces and spans. @@ -322,6 +344,22 @@ type CronConfig struct { RetryBaseDelay string `json:"retry_base_delay,omitempty"` // initial backoff delay (default "2s", Go duration) RetryMaxDelay string `json:"retry_max_delay,omitempty"` // maximum backoff delay (default "30s", Go duration) DefaultTimezone string `json:"default_timezone,omitempty"` // IANA timezone for cron expressions when not set per-job (e.g. "Asia/Ho_Chi_Minh") + JobTimeout string `json:"job_timeout,omitempty"` // max duration per cron job execution (default "10m", Go duration) +} + +// DefaultJobTimeout is the fallback timeout for cron job execution. +const DefaultJobTimeout = 10 * time.Minute + +// JobTimeoutDuration returns the configured job timeout or the default (10m). +func (cc CronConfig) JobTimeoutDuration() time.Duration { + if cc.JobTimeout != "" { + d, err := time.ParseDuration(cc.JobTimeout) + if err == nil && d > 0 { + return d + } + slog.Warn("cron: invalid job_timeout, using default", "value", cc.JobTimeout, "default", DefaultJobTimeout) + } + return DefaultJobTimeout } // ToRetryConfig converts CronConfig to cron.RetryConfig with defaults applied. diff --git a/internal/config/config_channels.go b/internal/config/config_channels.go index 0f31e5f1..037bc790 100644 --- a/internal/config/config_channels.go +++ b/internal/config/config_channels.go @@ -359,6 +359,8 @@ type GatewayConfig struct { BlockReply *bool `json:"block_reply,omitempty"` // deliver intermediate text during tool iterations (default false) ToolStatus *bool `json:"tool_status,omitempty"` // show tool name in streaming preview during tool execution (default true) TaskRecoveryIntervalSec int `json:"task_recovery_interval_sec,omitempty"` // team task recovery ticker interval in seconds (default 300 = 5min) + BackgroundProvider string `json:"background_provider,omitempty"` // LLM provider for background workers (vault enrichment, consolidation) + BackgroundModel string `json:"background_model,omitempty"` // LLM model for background workers } // ToolsConfig controls tool availability, policy, and web search. diff --git a/internal/config/config_system.go b/internal/config/config_system.go index 8027a9ea..c3e576aa 100644 --- a/internal/config/config_system.go +++ b/internal/config/config_system.go @@ -52,6 +52,10 @@ func (c *Config) ApplySystemConfigs(configs map[string]string) { boolean("gateway.tool_status", &c.Gateway.ToolStatus) integer("gateway.task_recovery_interval_sec", &c.Gateway.TaskRecoveryIntervalSec) + // Background workers (vault enrichment, consolidation) + str("background.provider", &c.Gateway.BackgroundProvider) + str("background.model", &c.Gateway.BackgroundModel) + // Tools str("tools.profile", &c.Tools.Profile) integer("tools.rate_limit_per_hour", &c.Tools.RateLimitPerHour) diff --git a/internal/consolidation/dreaming_config.go b/internal/consolidation/dreaming_config.go new file mode 100644 index 00000000..afe6b503 --- /dev/null +++ b/internal/consolidation/dreaming_config.go @@ -0,0 +1,84 @@ +package consolidation + +import ( + "context" + "time" + + "github.com/google/uuid" + "github.com/nextlevelbuilder/goclaw/internal/config" + "github.com/nextlevelbuilder/goclaw/internal/store" +) + +// resolvedDreamingConfig is the fully-populated view of dreaming settings used +// inside Handle(). All fields are concrete (no pointers) so the worker can +// operate without further nil-checks. +type resolvedDreamingConfig struct { + Enabled bool + Debounce time.Duration + Threshold int + VerboseLog bool +} + +// defaultDreamingConfig returns hardcoded defaults that match the worker's +// legacy behaviour before per-agent overrides were added. These values are +// mirrored into the struct defaults in workers.go. +func defaultDreamingConfig() resolvedDreamingConfig { + return resolvedDreamingConfig{ + Enabled: true, + Debounce: dreamingDefaultDebounce, + Threshold: dreamingDefaultThreshold, + VerboseLog: false, + } +} + +// mergeDreamingConfig applies an agent-provided override onto the defaults. +// Nil pointer fields fall through to defaults; zero integers are treated as +// "unset" so a legacy empty JSONB still resolves to defaults. Negative values +// are clamped to defaults to defend against malformed config. +func mergeDreamingConfig(base resolvedDreamingConfig, override *config.DreamingConfig) resolvedDreamingConfig { + if override == nil { + return base + } + if override.Enabled != nil { + base.Enabled = *override.Enabled + } + if override.DebounceMs > 0 { + base.Debounce = time.Duration(override.DebounceMs) * time.Millisecond + } + if override.Threshold > 0 { + base.Threshold = override.Threshold + } + if override.VerboseLog != nil { + base.VerboseLog = *override.VerboseLog + } + return base +} + +// DreamingConfigResolver fetches per-agent dreaming config at event handling +// time. Implementations should be fast and non-blocking; the consolidation +// worker calls this on every event that passes the agent-ID parse. +type DreamingConfigResolver func(ctx context.Context, agentID string) *config.DreamingConfig + +// newAgentStoreResolver builds a DreamingConfigResolver that reads from the +// given AgentStore and extracts MemoryConfig.Dreaming. Returns nil if the +// store is nil so the worker can fall back to defaults. +func newAgentStoreResolver(agents store.AgentCRUDStore) DreamingConfigResolver { + if agents == nil { + return nil + } + return func(ctx context.Context, agentID string) *config.DreamingConfig { + id, err := uuid.Parse(agentID) + if err != nil { + return nil + } + ag, err := agents.GetByIDUnscoped(ctx, id) + if err != nil || ag == nil { + return nil + } + mc := ag.ParseMemoryConfig() + if mc == nil { + return nil + } + return mc.Dreaming + } +} diff --git a/internal/consolidation/dreaming_config_test.go b/internal/consolidation/dreaming_config_test.go new file mode 100644 index 00000000..d533ccd4 --- /dev/null +++ b/internal/consolidation/dreaming_config_test.go @@ -0,0 +1,197 @@ +package consolidation + +import ( + "context" + "testing" + "time" + + "github.com/google/uuid" + "github.com/nextlevelbuilder/goclaw/internal/config" + "github.com/nextlevelbuilder/goclaw/internal/eventbus" + "github.com/nextlevelbuilder/goclaw/internal/providers" + "github.com/nextlevelbuilder/goclaw/internal/store" +) + +func boolPtr(b bool) *bool { return &b } + +func TestMergeDreamingConfigNilOverrideReturnsBase(t *testing.T) { + base := defaultDreamingConfig() + got := mergeDreamingConfig(base, nil) + if got != base { + t.Fatalf("nil override mutated base: got %+v, want %+v", got, base) + } +} + +func TestMergeDreamingConfigPartialOverride(t *testing.T) { + base := defaultDreamingConfig() + override := &config.DreamingConfig{ + Threshold: 3, + DebounceMs: 120_000, // 2 min + } + got := mergeDreamingConfig(base, override) + + if got.Threshold != 3 { + t.Errorf("Threshold = %d, want 3", got.Threshold) + } + if got.Debounce != 2*time.Minute { + t.Errorf("Debounce = %v, want 2m", got.Debounce) + } + // Enabled not set in override → stays at default. + if !got.Enabled { + t.Errorf("Enabled = false, want default true") + } +} + +func TestMergeDreamingConfigDisable(t *testing.T) { + base := defaultDreamingConfig() + override := &config.DreamingConfig{Enabled: boolPtr(false)} + got := mergeDreamingConfig(base, override) + if got.Enabled { + t.Errorf("Enabled = true, want false (override)") + } +} + +// TestMergeDreamingConfigVerboseLogNilPreservesDefault guards against the +// footgun where an override without VerboseLog silently reverts the base +// value. Pointer semantics require nil-check, not unconditional assignment. +func TestMergeDreamingConfigVerboseLogNilPreservesDefault(t *testing.T) { + base := defaultDreamingConfig() + base.VerboseLog = true // simulate an operator default of true + override := &config.DreamingConfig{Threshold: 2} + got := mergeDreamingConfig(base, override) + if !got.VerboseLog { + t.Errorf("VerboseLog = false, want true (nil override should preserve base)") + } +} + +func TestMergeDreamingConfigVerboseLogExplicitFalse(t *testing.T) { + base := defaultDreamingConfig() + base.VerboseLog = true + override := &config.DreamingConfig{VerboseLog: boolPtr(false)} + got := mergeDreamingConfig(base, override) + if got.VerboseLog { + t.Errorf("VerboseLog = true, want false (explicit override must apply)") + } +} + +func TestMergeDreamingConfigZeroFieldsIgnored(t *testing.T) { + // Zero values on non-pointer fields must not clobber defaults — this is + // critical for agents with an empty memory_config.dreaming JSONB object. + base := defaultDreamingConfig() + override := &config.DreamingConfig{} // all zero + got := mergeDreamingConfig(base, override) + if got.Threshold != base.Threshold { + t.Errorf("Threshold = %d, want base %d", got.Threshold, base.Threshold) + } + if got.Debounce != base.Debounce { + t.Errorf("Debounce = %v, want base %v", got.Debounce, base.Debounce) + } +} + +// TestDreamingWorkerHandleHonoursCustomThreshold verifies that a per-agent +// resolver lowering the threshold fires even when the worker's global default +// would have skipped the event. +func TestDreamingWorkerHandleHonoursCustomThreshold(t *testing.T) { + summaries := []store.EpisodicSummary{ + {ID: uuid.New(), Summary: "s1"}, + {ID: uuid.New(), Summary: "s2"}, + } + mockEpisodic := &mockEpisodicStore{ + countResult: 2, + unpromoted: summaries, + promoted: make(map[string]bool), + } + mockMemory := newMockMemoryStore() + mockProvider := &mockProvider{ + chatResp: &providers.ChatResponse{Content: "ok"}, + } + + worker := &dreamingWorker{ + episodicStore: mockEpisodic, + memoryStore: mockMemory, + provider: mockProvider, + model: "test", + threshold: 5, // global default says skip at count=2 + debounce: 1 * time.Second, + resolveConfig: func(_ context.Context, _ string) *config.DreamingConfig { + return &config.DreamingConfig{Threshold: 2} // per-agent override lowers it + }, + } + + err := worker.Handle(context.Background(), eventbus.DomainEvent{ + Type: eventbus.EventEpisodicCreated, + TenantID: uuid.New().String(), + AgentID: "agent-123", + UserID: "user-123", + Payload: &eventbus.EpisodicCreatedPayload{}, + }) + if err != nil { + t.Fatalf("Handle failed: %v", err) + } + if len(mockEpisodic.promoted) != 2 { + t.Errorf("Expected 2 promotions with override threshold, got %d", len(mockEpisodic.promoted)) + } +} + +// TestDreamingWorkerHandleDisabledSkips verifies that Enabled=false prevents +// any store activity, regardless of threshold/debounce. +func TestDreamingWorkerHandleDisabledSkips(t *testing.T) { + mockEpisodic := &mockEpisodicStore{ + countResult: 100, // would normally trigger + promoted: make(map[string]bool), + } + + worker := &dreamingWorker{ + episodicStore: mockEpisodic, + threshold: 5, + debounce: 1 * time.Second, + resolveConfig: func(_ context.Context, _ string) *config.DreamingConfig { + return &config.DreamingConfig{Enabled: boolPtr(false)} + }, + } + + err := worker.Handle(context.Background(), eventbus.DomainEvent{ + Type: eventbus.EventEpisodicCreated, + TenantID: uuid.New().String(), + AgentID: "agent-disabled", + UserID: "user-123", + Payload: &eventbus.EpisodicCreatedPayload{}, + }) + if err != nil { + t.Fatalf("Handle failed: %v", err) + } + if len(mockEpisodic.promoted) > 0 { + t.Errorf("Expected 0 promotions when disabled, got %d", len(mockEpisodic.promoted)) + } + // CountUnpromoted must also be skipped — it's a downstream read that + // dreaming workers should not perform when an agent opts out. + if mockEpisodic.countCalls > 0 { + t.Errorf("Expected 0 CountUnpromoted calls when disabled, got %d", mockEpisodic.countCalls) + } +} + +// TestDreamingWorkerHandleNilResolverUsesDefaults verifies backward +// compatibility — no resolver means the worker behaves exactly as before. +func TestDreamingWorkerHandleNilResolverUsesDefaults(t *testing.T) { + mockEpisodic := &mockEpisodicStore{countResult: 2} // below default 5 + worker := &dreamingWorker{ + episodicStore: mockEpisodic, + threshold: 5, + debounce: 1 * time.Second, + // resolveConfig: nil + } + err := worker.Handle(context.Background(), eventbus.DomainEvent{ + Type: eventbus.EventEpisodicCreated, + TenantID: uuid.New().String(), + AgentID: "agent-123", + UserID: "user-123", + Payload: &eventbus.EpisodicCreatedPayload{}, + }) + if err != nil { + t.Fatalf("Handle failed: %v", err) + } + // Below threshold — no promotions expected (default behaviour preserved). + if len(mockEpisodic.promoted) > 0 { + t.Errorf("Expected no promotions below default threshold, got %d", len(mockEpisodic.promoted)) + } +} diff --git a/internal/consolidation/dreaming_scoring_test.go b/internal/consolidation/dreaming_scoring_test.go new file mode 100644 index 00000000..c42e3d58 --- /dev/null +++ b/internal/consolidation/dreaming_scoring_test.go @@ -0,0 +1,191 @@ +package consolidation + +import ( + "context" + "testing" + "time" + + "github.com/google/uuid" + "github.com/nextlevelbuilder/goclaw/internal/eventbus" + "github.com/nextlevelbuilder/goclaw/internal/providers" + "github.com/nextlevelbuilder/goclaw/internal/store" +) + +// TestDreamingWorkerUsesScoredListing verifies the worker fetches via +// ListUnpromotedScored (not the legacy created_at ASC path) so recall-hot +// entries synthesise first. +func TestDreamingWorkerUsesScoredListing(t *testing.T) { + // Two unpromoted entries: one cold-but-old, one hot-recent. + now := time.Now().UTC() + lastRecall := now.Add(-1 * time.Hour) + summaries := []store.EpisodicSummary{ + {ID: uuid.New(), Summary: "hot", RecallCount: 5, RecallScore: 0.9, LastRecalledAt: &lastRecall, CreatedAt: now.Add(-2 * 24 * time.Hour)}, + {ID: uuid.New(), Summary: "cold", CreatedAt: now.Add(-5 * 24 * time.Hour)}, + } + // Custom scored-mock flagging whether the correct listing method fired. + mockEp := &scoredEpisodicMock{ + mockEpisodicStore: mockEpisodicStore{ + countResult: 5, + unpromoted: summaries, + promoted: make(map[string]bool), + }, + } + mockMem := newMockMemoryStore() + mockProv := &mockProvider{chatResp: &providers.ChatResponse{Content: "consolidated"}} + + worker := &dreamingWorker{ + episodicStore: mockEp, + memoryStore: mockMem, + provider: mockProv, + model: "test", + threshold: 5, + debounce: 1 * time.Second, + } + + err := worker.Handle(context.Background(), eventbus.DomainEvent{ + Type: eventbus.EventEpisodicCreated, + TenantID: uuid.New().String(), + AgentID: "agent-scored", + UserID: "user-scored", + Payload: &eventbus.EpisodicCreatedPayload{}, + }) + if err != nil { + t.Fatalf("Handle failed: %v", err) + } + if !mockEp.scoredCalled { + t.Errorf("worker did not call ListUnpromotedScored (scoring bypass)") + } + if mockEp.legacyCalled { + t.Errorf("worker called legacy ListUnpromoted — should use scored path") + } + if len(mockEp.promoted) != 2 { + t.Errorf("expected 2 entries promoted, got %d", len(mockEp.promoted)) + } +} + +// TestDreamingWorkerFiltersBelowThreshold verifies that even when the agent +// has enough unpromoted entries to pass the count check, the recall-score +// filter can still skip synthesis if every entry is too weak. +func TestDreamingWorkerFiltersBelowThreshold(t *testing.T) { + now := time.Now().UTC() + // All entries recalled once (below MinRecallCount=2) with trivial score. + summaries := make([]store.EpisodicSummary, 5) + for i := range summaries { + summaries[i] = store.EpisodicSummary{ + ID: uuid.New(), + Summary: "weak", + CreatedAt: now.Add(-30 * 24 * time.Hour), + RecallCount: 1, + RecallScore: 0.1, + } + } + mockEp := &scoredEpisodicMock{ + mockEpisodicStore: mockEpisodicStore{ + countResult: 5, + unpromoted: summaries, + promoted: make(map[string]bool), + }, + } + mockMem := newMockMemoryStore() + mockProv := &mockProvider{chatResp: &providers.ChatResponse{Content: "noop"}} + + worker := &dreamingWorker{ + episodicStore: mockEp, + memoryStore: mockMem, + provider: mockProv, + model: "test", + threshold: 5, + debounce: 1 * time.Second, + } + + err := worker.Handle(context.Background(), eventbus.DomainEvent{ + Type: eventbus.EventEpisodicCreated, + TenantID: uuid.New().String(), + AgentID: "agent-filter", + UserID: "user-filter", + Payload: &eventbus.EpisodicCreatedPayload{}, + }) + if err != nil { + t.Fatalf("Handle failed: %v", err) + } + if len(mockEp.promoted) > 0 { + t.Errorf("expected all entries filtered, got %d promoted", len(mockEp.promoted)) + } + if len(mockMem.docs) > 0 { + t.Errorf("expected no synthesis document, got %d", len(mockMem.docs)) + } +} + +// TestDreamingWorkerFilterEmptyStampsDebounce guards against the starvation +// loop: when all entries fail the recall threshold filter we still bump +// lastRun, otherwise every subsequent episodic.created event re-runs the +// full scoring pipeline in a tight loop. See code review note P10.1. +func TestDreamingWorkerFilterEmptyStampsDebounce(t *testing.T) { + now := time.Now().UTC() + summaries := make([]store.EpisodicSummary, 5) + for i := range summaries { + summaries[i] = store.EpisodicSummary{ + ID: uuid.New(), + Summary: "weak", + CreatedAt: now.Add(-30 * 24 * time.Hour), + RecallCount: 1, // below MinRecallCount=2 → filtered + RecallScore: 0.05, + } + } + mockEp := &scoredEpisodicMock{ + mockEpisodicStore: mockEpisodicStore{ + countResult: 5, + unpromoted: summaries, + promoted: make(map[string]bool), + }, + } + worker := &dreamingWorker{ + episodicStore: mockEp, + memoryStore: newMockMemoryStore(), + provider: &mockProvider{chatResp: &providers.ChatResponse{Content: "noop"}}, + model: "test", + threshold: 5, + debounce: 10 * time.Minute, // realistic + } + + ev := eventbus.DomainEvent{ + Type: eventbus.EventEpisodicCreated, + TenantID: uuid.New().String(), + AgentID: "agent-loop", + UserID: "user-loop", + Payload: &eventbus.EpisodicCreatedPayload{}, + } + + // First event: filter drops everything → no promotions. + if err := worker.Handle(context.Background(), ev); err != nil { + t.Fatalf("first Handle: %v", err) + } + firstCountCalls := mockEp.countCalls + + // Second event immediately after: debounce should skip BEFORE CountUnpromoted. + if err := worker.Handle(context.Background(), ev); err != nil { + t.Fatalf("second Handle: %v", err) + } + if mockEp.countCalls != firstCountCalls { + t.Errorf("debounce not stamped on filter-empty skip — CountUnpromoted called %d times, want %d", + mockEp.countCalls, firstCountCalls) + } +} + +// scoredEpisodicMock wraps mockEpisodicStore with tracking flags for the +// two ListUnpromoted* methods so tests can assert which path the worker used. +type scoredEpisodicMock struct { + mockEpisodicStore + scoredCalled bool + legacyCalled bool +} + +func (m *scoredEpisodicMock) ListUnpromoted(ctx context.Context, agentID, userID string, limit int) ([]store.EpisodicSummary, error) { + m.legacyCalled = true + return m.mockEpisodicStore.ListUnpromoted(ctx, agentID, userID, limit) +} + +func (m *scoredEpisodicMock) ListUnpromotedScored(ctx context.Context, agentID, userID string, limit int) ([]store.EpisodicSummary, error) { + m.scoredCalled = true + return m.mockEpisodicStore.ListUnpromotedScored(ctx, agentID, userID, limit) +} diff --git a/internal/consolidation/dreaming_worker.go b/internal/consolidation/dreaming_worker.go index 519363b7..7eaf46de 100644 --- a/internal/consolidation/dreaming_worker.go +++ b/internal/consolidation/dreaming_worker.go @@ -27,10 +27,56 @@ type dreamingWorker struct { episodicStore store.EpisodicStore memoryStore store.MemoryStore provider providers.Provider - model string // LLM model for synthesis - threshold int // min unpromoted entries before running - debounce time.Duration // min interval between runs per agent/user - lastRun sync.Map // key: "agentID:userID" → time.Time + model string // LLM model for synthesis + + // threshold/debounce are the global defaults. Per-agent overrides come + // from resolveConfig which reads the agent's MemoryConfig.Dreaming JSONB. + threshold int + debounce time.Duration + resolveConfig DreamingConfigResolver + + lastRun sync.Map // key: "agentID:userID" → time.Time +} + +// formatEntryForSynthesis renders a single episodic entry with recall +// metadata for the LLM synthesis prompt. Entries with recall signal are +// tagged so the LLM can weight them higher; unrecalled entries pass through +// as plain summaries. +func formatEntryForSynthesis(e store.EpisodicSummary) string { + if e.RecallCount == 0 { + return e.Summary + } + lastRecall := "never" + if e.LastRecalledAt != nil { + lastRecall = e.LastRecalledAt.Format("Jan 2") + } + return fmt.Sprintf("[recalled %dx, last: %s] %s", e.RecallCount, lastRecall, e.Summary) +} + +// logSkip emits dreaming skip reasons at debug level by default, elevating +// to info when verbose logging is enabled via per-agent config. +func logSkip(verbose bool, msg string, args ...any) { + if verbose { + slog.Info(msg, args...) + return + } + slog.Debug(msg, args...) +} + +// effectiveConfig merges the worker's defaults with any per-agent override. +// Centralised so tests can reason about the precedence in one place. +func (w *dreamingWorker) effectiveConfig(ctx context.Context, agentID string) resolvedDreamingConfig { + base := defaultDreamingConfig() + if w.threshold > 0 { + base.Threshold = w.threshold + } + if w.debounce > 0 { + base.Debounce = w.debounce + } + if w.resolveConfig == nil { + return base + } + return mergeDreamingConfig(base, w.resolveConfig(ctx, agentID)) } // Handle processes an episodic.created event for the dreaming pipeline. @@ -45,41 +91,55 @@ func (w *dreamingWorker) Handle(ctx context.Context, event eventbus.DomainEvent) agentID := event.AgentID userID := event.UserID + // Resolve per-agent config (threshold/debounce/enabled). Falls back to + // struct defaults when no override is stored on the agent. + cfg := w.effectiveConfig(ctx, agentID) + if !cfg.Enabled { + if cfg.VerboseLog { + slog.Info("dreaming: disabled for agent", "agent", agentID, "user", userID) + } + return nil + } + // Debounce: skip if ran recently for this pair. key := agentID + ":" + userID - debounce := w.debounce - if debounce == 0 { - debounce = dreamingDefaultDebounce - } if v, ok := w.lastRun.Load(key); ok { - if last, ok := v.(time.Time); ok && time.Since(last) < debounce { - slog.Debug("dreaming: debounce skip", "agent", agentID, "user", userID) + if last, ok := v.(time.Time); ok && time.Since(last) < cfg.Debounce { + logSkip(cfg.VerboseLog, "dreaming: debounce skip", "agent", agentID, "user", userID) return nil } } // Count unpromoted entries; skip if below threshold. - threshold := w.threshold - if threshold <= 0 { - threshold = dreamingDefaultThreshold - } count, err := w.episodicStore.CountUnpromoted(ctx, agentID, userID) if err != nil { slog.Warn("dreaming: count unpromoted failed", "err", err, "agent", agentID) return nil } - if count < threshold { - slog.Debug("dreaming: below threshold", "count", count, "threshold", threshold, "agent", agentID) + if count < cfg.Threshold { + logSkip(cfg.VerboseLog, "dreaming: below threshold", "count", count, "threshold", cfg.Threshold, "agent", agentID) return nil } - // Fetch unpromoted entries. - entries, err := w.episodicStore.ListUnpromoted(ctx, agentID, userID, dreamingFetchLimit) + // Fetch unpromoted entries ordered by recall_score DESC (Phase 10). Falls + // back to created_at ASC for ties so agents with no recall history still + // get oldest-first behaviour. + entries, err := w.episodicStore.ListUnpromotedScored(ctx, agentID, userID, dreamingFetchLimit) if err != nil { slog.Warn("dreaming: list unpromoted failed", "err", err, "agent", agentID) return nil } + // Filter by recall-score thresholds before synthesis so weak entries + // don't burn LLM tokens. Never-recalled entries bypass the count minimum + // via the freshness component in ComputeRecallScore. + entries = filterByRecallThresholds(entries, defaultRecallThresholds(), time.Now().UTC()) if len(entries) == 0 { + // Stamp lastRun even on empty-filter skips — otherwise every subsequent + // episodic.created event re-runs CountUnpromoted + ListUnpromotedScored + // + filter in a tight loop until the agent accumulates fresh content. + // See code review note P10.1. + w.lastRun.Store(key, time.Now()) + logSkip(cfg.VerboseLog, "dreaming: all entries below recall thresholds", "agent", agentID, "user", userID) return nil } @@ -119,10 +179,12 @@ func (w *dreamingWorker) Handle(ctx context.Context, event eventbus.DomainEvent) } // synthesize calls the LLM to extract long-term facts from session summaries. +// Each entry is annotated with its recall metadata so the LLM can weight +// frequently-recalled memories higher during synthesis. func (w *dreamingWorker) synthesize(ctx context.Context, entries []store.EpisodicSummary) (string, error) { summaries := make([]string, len(entries)) for i, e := range entries { - summaries[i] = e.Summary + summaries[i] = formatEntryForSynthesis(e) } body := strings.Join(summaries, "\n---\n") diff --git a/internal/consolidation/scoring.go b/internal/consolidation/scoring.go new file mode 100644 index 00000000..1e74245a --- /dev/null +++ b/internal/consolidation/scoring.go @@ -0,0 +1,128 @@ +package consolidation + +import ( + "math" + "time" + + "github.com/nextlevelbuilder/goclaw/internal/store" +) + +// Scoring constants. Chosen to match the plan's 4-component formula (plan has +// 6 components but GoClaw lacks per-episode conceptual + consolidation data). +// +// score = 0.30*frequency + 0.35*relevance + 0.20*recency + 0.15*freshness +// +// halfLifeDays=14 matches typical short-term memory retention windows; tuning +// should be informed by operator feedback once real recall data accumulates. +const ( + recallWeightFrequency = 0.30 + recallWeightRelevance = 0.35 + recallWeightRecency = 0.20 + recallWeightFreshness = 0.15 + recallHalfLifeDays = 14.0 + // frequencyCap: recall_count above this yields max frequency component + // (avoids runaway-frequency bias for hot entries). + recallFrequencyCap = 10 +) + +// recallThresholds holds the minimum scoring cutoffs applied to +// ListUnpromotedScored results before LLM synthesis. Chosen well below the +// TS reference (0.75 / 3) because GoClaw boots with zero recall data — +// early-life agents must still reach synthesis via the freshness component. +type recallThresholds struct { + MinScore float64 + MinRecallCount int +} + +func defaultRecallThresholds() recallThresholds { + return recallThresholds{MinScore: 0.2, MinRecallCount: 2} +} + +// ComputeRecallScore computes the weighted recall score for an episodic entry. +// Pure function — no IO, no randomness — so the dreaming worker can compute +// scores on demand instead of materialising them into the DB on every event. +// +// Returned value is in [0, 1+ε]. In practice results cluster in [0, 0.8]; +// thresholds in defaultRecallThresholds are calibrated to that range. +func ComputeRecallScore(entry store.EpisodicSummary, now time.Time) float64 { + // 1. Frequency (30%): log-normalised recall count. Log1p smooths the + // curve so 0 → 0, 1 → 0.29, 10 → 1.0; anything above cap stays at 1. + freq := math.Log1p(float64(entry.RecallCount)) / math.Log1p(recallFrequencyCap) + if freq > 1.0 { + freq = 1.0 + } + + // 2. Relevance (35%): running average of memory_search hit scores. + // Already lives in recall_score column and is clamped server-side to [0,1] + // by RecordRecall callers. Defensive clamp here for robustness. + rel := entry.RecallScore + if rel < 0 { + rel = 0 + } else if rel > 1 { + rel = 1 + } + + // Exponential decay shared by recency + freshness components. + // lambda = ln(2) / halfLife → score halves every `recallHalfLifeDays` days. + lambda := math.Ln2 / recallHalfLifeDays + + // 3. Recency (20%): decay from LastRecalledAt (falls back to CreatedAt + // for entries that have never been searched). Entries recalled today + // score ~1.0; 14 days ago → 0.5; 28 days ago → 0.25. + lastRef := entry.CreatedAt + if entry.LastRecalledAt != nil { + lastRef = *entry.LastRecalledAt + } + recency := decayFrom(lastRef, now, lambda) + + // 4. Freshness (15%): decay from CreatedAt regardless of recall history. + // Keeps brand-new entries eligible even with zero recall — prevents cold + // start from starving the synthesis pipeline. + freshness := decayFrom(entry.CreatedAt, now, lambda) + + return recallWeightFrequency*freq + + recallWeightRelevance*rel + + recallWeightRecency*recency + + recallWeightFreshness*freshness +} + +// decayFrom returns exp(-lambda * ageDays). Negative ages (clock skew, +// future-dated rows) clamp to 1.0 so the test fixtures are predictable. +func decayFrom(then, now time.Time, lambda float64) float64 { + if then.IsZero() || !then.Before(now) { + return 1.0 + } + ageDays := now.Sub(then).Hours() / 24 + return math.Exp(-lambda * ageDays) +} + +// filterByRecallThresholds drops entries whose recall signal is too weak +// to justify LLM synthesis. Pure function so the dreaming worker can +// sort-then-filter in a single pass. +// +// Bootstrap-friendly rules: +// - Never-recalled entries (RecallCount == 0) bypass BOTH thresholds. +// Freshness + recency keep them eligible so new agents can synthesise +// their earliest sessions without any memory_search activity. +// - Entries that HAVE been recalled must clear both MinRecallCount and +// MinScore. If an entry has seen 1 weak hit, it's filtered. +func filterByRecallThresholds(entries []store.EpisodicSummary, th recallThresholds, now time.Time) []store.EpisodicSummary { + if len(entries) == 0 { + return entries + } + kept := entries[:0] + for _, e := range entries { + if e.RecallCount == 0 { + kept = append(kept, e) + continue + } + if e.RecallCount < th.MinRecallCount { + continue + } + if ComputeRecallScore(e, now) < th.MinScore { + continue + } + kept = append(kept, e) + } + return kept +} diff --git a/internal/consolidation/scoring_test.go b/internal/consolidation/scoring_test.go new file mode 100644 index 00000000..8e3e4bb9 --- /dev/null +++ b/internal/consolidation/scoring_test.go @@ -0,0 +1,160 @@ +package consolidation + +import ( + "math" + "strings" + "testing" + "time" + + "github.com/google/uuid" + "github.com/nextlevelbuilder/goclaw/internal/store" +) + +// floatEq checks two floats are equal within epsilon for the scoring assertions. +func floatEq(a, b, eps float64) bool { return math.Abs(a-b) < eps } + +func TestComputeRecallScoreZeroRecallFreshEntry(t *testing.T) { + now := time.Date(2026, 4, 10, 12, 0, 0, 0, time.UTC) + entry := store.EpisodicSummary{ + ID: uuid.New(), + CreatedAt: now, // brand new, zero recalls + } + // freq=0, rel=0, recency=1.0, freshness=1.0 + // 0.30*0 + 0.35*0 + 0.20*1 + 0.15*1 = 0.35 + score := ComputeRecallScore(entry, now) + if !floatEq(score, 0.35, 0.001) { + t.Errorf("score = %f, want ~0.35", score) + } +} + +func TestComputeRecallScoreOldUnrecalled(t *testing.T) { + now := time.Date(2026, 4, 10, 12, 0, 0, 0, time.UTC) + entry := store.EpisodicSummary{ + CreatedAt: now.AddDate(0, 0, -28), // 28 days old, never recalled + } + // recency = freshness = exp(-ln2/14 * 28) = exp(-ln2*2) = 0.25 + // score = 0 + 0 + 0.2*0.25 + 0.15*0.25 = 0.0875 + score := ComputeRecallScore(entry, now) + if !floatEq(score, 0.0875, 0.001) { + t.Errorf("score = %f, want ~0.0875", score) + } +} + +func TestComputeRecallScoreExponentialDecayHalfLife(t *testing.T) { + now := time.Date(2026, 4, 10, 12, 0, 0, 0, time.UTC) + // Entry recalled exactly 14 days ago (one half-life). Recency should halve. + lastRecall := now.AddDate(0, 0, -14) + entry := store.EpisodicSummary{ + CreatedAt: lastRecall, // created at same moment it was last recalled + RecallCount: 1, + RecallScore: 1.0, + LastRecalledAt: &lastRecall, + } + // freq = log1p(1) / log1p(10) = 0.693/2.398 ≈ 0.289 + // rel = 1.0 + // recency = freshness = exp(-ln2) = 0.5 + // score = 0.30*0.289 + 0.35*1.0 + 0.20*0.5 + 0.15*0.5 = 0.0867 + 0.35 + 0.175 = 0.6117 + score := ComputeRecallScore(entry, now) + if !floatEq(score, 0.6117, 0.001) { + t.Errorf("score = %f, want ~0.6117", score) + } +} + +func TestComputeRecallScoreHighRecallSignal(t *testing.T) { + now := time.Date(2026, 4, 10, 12, 0, 0, 0, time.UTC) + entry := store.EpisodicSummary{ + CreatedAt: now.AddDate(0, 0, -3), // 3 days old + RecallCount: 10, + RecallScore: 0.9, + LastRecalledAt: &now, // recalled today + } + // freq ≈ 1.0 (cap) + // rel = 0.9 + // recency ≈ 1.0 (lastRef=now) + // freshness = exp(-ln2/14 * 3) ≈ 0.862 + // score ≈ 0.30 + 0.315 + 0.20 + 0.1293 ≈ 0.944 + score := ComputeRecallScore(entry, now) + if score < 0.9 { + t.Errorf("score = %f, want >0.9 (hot entry)", score) + } +} + +func TestComputeRecallScoreClampsRelevance(t *testing.T) { + now := time.Date(2026, 4, 10, 12, 0, 0, 0, time.UTC) + // Malformed entry with out-of-range recall_score — scoring must clamp. + entry := store.EpisodicSummary{ + CreatedAt: now, + RecallCount: 1, + RecallScore: 2.5, // bug: > 1.0 + } + score := ComputeRecallScore(entry, now) + if score > 1.1 { + t.Errorf("score = %f exceeds reasonable bounds (relevance should clamp)", score) + } +} + +func TestFilterByRecallThresholdsFreshBypass(t *testing.T) { + now := time.Date(2026, 4, 10, 12, 0, 0, 0, time.UTC) + entries := []store.EpisodicSummary{ + // Never recalled, 28 days old → low score but bypasses via RecallCount=0. + {ID: uuid.New(), CreatedAt: now.AddDate(0, 0, -28)}, + } + kept := filterByRecallThresholds(entries, recallThresholds{MinScore: 0.5, MinRecallCount: 2}, now) + if len(kept) != 1 { + t.Errorf("expected 1 kept (fresh bypass), got %d", len(kept)) + } +} + +func TestFilterByRecallThresholdsPreservesOrder(t *testing.T) { + // In-place filter must not scramble the caller's ORDER BY recall_score DESC. + now := time.Date(2026, 4, 10, 12, 0, 0, 0, time.UTC) + entries := []store.EpisodicSummary{ + {ID: uuid.MustParse("00000000-0000-0000-0000-000000000001"), CreatedAt: now, RecallCount: 10, RecallScore: 0.9, LastRecalledAt: &now}, + {ID: uuid.MustParse("00000000-0000-0000-0000-000000000002"), CreatedAt: now, RecallCount: 1, RecallScore: 0.1}, // filtered + {ID: uuid.MustParse("00000000-0000-0000-0000-000000000003"), CreatedAt: now, RecallCount: 5, RecallScore: 0.7, LastRecalledAt: &now}, + } + kept := filterByRecallThresholds(entries, defaultRecallThresholds(), now) + if len(kept) != 2 { + t.Fatalf("len=%d, want 2", len(kept)) + } + if kept[0].ID.String() != "00000000-0000-0000-0000-000000000001" || + kept[1].ID.String() != "00000000-0000-0000-0000-000000000003" { + t.Errorf("order corrupted: %v", []string{kept[0].ID.String(), kept[1].ID.String()}) + } +} + +func TestFilterByRecallThresholdsDropsWeakRecalled(t *testing.T) { + now := time.Date(2026, 4, 10, 12, 0, 0, 0, time.UTC) + entries := []store.EpisodicSummary{ + // Recalled once — below MinRecallCount=2 → filtered. + {ID: uuid.New(), CreatedAt: now, RecallCount: 1, RecallScore: 0.9}, + // Recalled 3 times with good score — kept. + {ID: uuid.New(), CreatedAt: now, RecallCount: 3, RecallScore: 0.8, LastRecalledAt: &now}, + // Never recalled — kept via freshness. + {ID: uuid.New(), CreatedAt: now}, + } + kept := filterByRecallThresholds(entries, defaultRecallThresholds(), now) + if len(kept) != 2 { + t.Errorf("expected 2 kept, got %d", len(kept)) + } +} + +func TestFormatEntryForSynthesisAnnotatesRecalled(t *testing.T) { + now := time.Date(2026, 4, 10, 12, 0, 0, 0, time.UTC) + entry := store.EpisodicSummary{ + Summary: "test session", + RecallCount: 5, + LastRecalledAt: &now, + } + out := formatEntryForSynthesis(entry) + if !strings.Contains(out, "recalled 5x") || !strings.Contains(out, "test session") { + t.Errorf("output missing metadata or summary: %q", out) + } +} + +func TestFormatEntryForSynthesisBypassesUnrecalled(t *testing.T) { + entry := store.EpisodicSummary{Summary: "fresh content"} + if out := formatEntryForSynthesis(entry); out != "fresh content" { + t.Errorf("unrecalled entry should pass through unchanged, got %q", out) + } +} diff --git a/internal/consolidation/workers.go b/internal/consolidation/workers.go index cf5c8b12..903cfc92 100644 --- a/internal/consolidation/workers.go +++ b/internal/consolidation/workers.go @@ -24,6 +24,10 @@ type ConsolidationDeps struct { Provider providers.Provider // for LLM summarization Model string Extractor EntityExtractor + // AgentStore is optional: when present, the dreaming worker reads + // per-agent overrides from MemoryConfig.Dreaming. If nil, the worker + // uses its built-in defaults for every agent. + AgentStore store.AgentCRUDStore } // Register wires all consolidation workers to the event bus. @@ -52,6 +56,7 @@ func Register(deps ConsolidationDeps) func() { model: deps.Model, threshold: dreamingDefaultThreshold, debounce: dreamingDefaultDebounce, + resolveConfig: newAgentStoreResolver(deps.AgentStore), } unsub1 := deps.EventBus.Subscribe(eventbus.EventSessionCompleted, episodic.Handle) diff --git a/internal/consolidation/workers_test.go b/internal/consolidation/workers_test.go index 6d816be3..14045dbb 100644 --- a/internal/consolidation/workers_test.go +++ b/internal/consolidation/workers_test.go @@ -15,16 +15,25 @@ import ( "github.com/nextlevelbuilder/goclaw/internal/store" ) +// recallCall is a single RecordRecall invocation captured by the mock store +// so tests can assert the (id, score) stream forwarded by memory_search. +type recallCall struct { + ID string + Score float64 +} + // mockEpisodicStore implements store.EpisodicStore for testing. type mockEpisodicStore struct { - created []*store.EpisodicSummary - existsByID map[string]bool - unpromoted []store.EpisodicSummary - promoted map[string]bool - countResult int - pruneErr error - pruneCount int - mu sync.Mutex + created []*store.EpisodicSummary + existsByID map[string]bool + unpromoted []store.EpisodicSummary + promoted map[string]bool + countResult int + countCalls int // set by CountUnpromoted — used by disabled-skip tests + recallCalls []recallCall + pruneErr error + pruneCount int + mu sync.Mutex } func (m *mockEpisodicStore) Create(_ context.Context, ep *store.EpisodicSummary) error { @@ -68,6 +77,7 @@ func (m *mockEpisodicStore) PruneExpired(_ context.Context) (int, error) { func (m *mockEpisodicStore) CountUnpromoted(_ context.Context, _, _ string) (int, error) { m.mu.Lock() defer m.mu.Unlock() + m.countCalls++ return m.countResult, nil } @@ -77,6 +87,23 @@ func (m *mockEpisodicStore) ListUnpromoted(_ context.Context, _, _ string, _ int return m.unpromoted, nil } +// ListUnpromotedScored returns the same fixture as ListUnpromoted; tests that +// need to verify sort order override the method directly on a dedicated mock. +func (m *mockEpisodicStore) ListUnpromotedScored(_ context.Context, _, _ string, _ int) ([]store.EpisodicSummary, error) { + m.mu.Lock() + defer m.mu.Unlock() + return m.unpromoted, nil +} + +// RecordRecall appends the call into the recallCalls slice so tests can +// assert which episodic IDs received which score. +func (m *mockEpisodicStore) RecordRecall(_ context.Context, id string, score float64) error { + m.mu.Lock() + defer m.mu.Unlock() + m.recallCalls = append(m.recallCalls, recallCall{ID: id, Score: score}) + return nil +} + func (m *mockEpisodicStore) MarkPromoted(_ context.Context, ids []string) error { m.mu.Lock() defer m.mu.Unlock() diff --git a/internal/cron/service_execution.go b/internal/cron/service_execution.go index 4e299058..5af8c790 100644 --- a/internal/cron/service_execution.go +++ b/internal/cron/service_execution.go @@ -6,7 +6,6 @@ import ( "log/slog" "os" "path/filepath" - "sync" "time" "github.com/adhocore/gronx" @@ -146,11 +145,22 @@ func (cs *Service) runLoop(stopChan chan struct{}) { case <-stopChan: return case <-ticker.C: - cs.checkJobs() + cs.safeCheckJobs() } } } +// safeCheckJobs wraps checkJobs with panic recovery so a panic in any +// check/claim logic doesn't kill the runLoop goroutine. +func (cs *Service) safeCheckJobs() { + defer func() { + if r := recover(); r != nil { + slog.Error("cron: checkJobs panicked — runLoop continues", "panic", fmt.Sprint(r)) + } + }() + cs.checkJobs() +} + func (cs *Service) checkJobs() { cs.mu.Lock() @@ -189,17 +199,16 @@ func (cs *Service) checkJobs() { cs.saveUnsafe() cs.mu.Unlock() - // Execute jobs in parallel — scheduler enforces per-session serialization - var wg sync.WaitGroup + // Execute jobs in parallel without blocking the runLoop. + // Previously wg.Wait() blocked here — if any job hung (e.g. LLM timeout, + // agent loop stuck), the entire cron scheduler would stop checking for new + // due jobs. Now each job runs independently with panic recovery. for _, dj := range dueJobs { - wg.Add(1) go func(id string, scheduledAtMS int64) { - defer wg.Done() defer safego.Recover(nil, "job_id", id) cs.executeJobByID(id, scheduledAtMS) }(dj.id, dj.scheduledAtMS) } - wg.Wait() } func (cs *Service) executeJobByID(jobID string, scheduledAtMS int64) { diff --git a/internal/gateway/methods/tenants.go b/internal/gateway/methods/tenants.go index 7aff4966..fd57465b 100644 --- a/internal/gateway/methods/tenants.go +++ b/internal/gateway/methods/tenants.go @@ -381,10 +381,26 @@ func (m *TenantsMethods) handleMine(ctx context.Context, client *gateway.Client, return } + // Batch-fetch all tenant data in a single query instead of per-membership. + ids := make([]uuid.UUID, 0, len(memberships)) + for _, mem := range memberships { + ids = append(ids, mem.TenantID) + } + tenants, tErr := m.tenantStore.GetTenantsByIDs(ctx, ids) + if tErr != nil { + slog.Error("tenants.mine: batch fetch failed", "error", tErr, "user_id", userID) + client.SendResponse(protocol.NewErrorResponse(req.ID, protocol.ErrInternal, i18n.T(locale, i18n.MsgFailedToList, "tenants"))) + return + } + tenantMap := make(map[uuid.UUID]*store.TenantData, len(tenants)) + for i := range tenants { + tenantMap[tenants[i].ID] = &tenants[i] + } + entries := make([]tenantEntry, 0, len(memberships)) for _, mem := range memberships { - t, err := m.tenantStore.GetTenant(ctx, mem.TenantID) - if err != nil || t == nil || t.Status != store.TenantStatusActive { + t := tenantMap[mem.TenantID] + if t == nil || t.Status != store.TenantStatusActive { continue } entries = append(entries, tenantEntry{ID: t.ID.String(), Name: t.Name, Slug: t.Slug, Role: mem.Role, Status: t.Status}) diff --git a/internal/gateway/methods/usage.go b/internal/gateway/methods/usage.go index e779a79c..2e817683 100644 --- a/internal/gateway/methods/usage.go +++ b/internal/gateway/methods/usage.go @@ -50,28 +50,28 @@ func (m *UsageMethods) handleGet(ctx context.Context, client *gateway.Client, re params.Limit = 20 } - sessions := m.sessions.List(ctx, params.AgentID) + // Use ListPagedRich: single query returns model, provider, tokens — no N+1 GetOrCreate loop. + // Fetch large batch to filter non-zero tokens, then paginate in-memory. + result := m.sessions.ListPagedRich(ctx, store.SessionListOpts{ + AgentID: params.AgentID, + Limit: 10000, + }) - records := make([]UsageRecord, 0, len(sessions)) - for _, s := range sessions { - // Get full session data for token info - data := m.sessions.GetOrCreate(ctx, s.Key) - if data.InputTokens == 0 && data.OutputTokens == 0 { + records := make([]UsageRecord, 0, len(result.Sessions)) + for _, s := range result.Sessions { + if s.InputTokens == 0 && s.OutputTokens == 0 { continue } - - // Extract agentID from session key (format: "agent::") agentID := extractAgentIDFromKey(s.Key) - records = append(records, UsageRecord{ AgentID: agentID, SessionKey: s.Key, - Model: data.Model, - Provider: data.Provider, - InputTokens: data.InputTokens, - OutputTokens: data.OutputTokens, - TotalTokens: data.InputTokens + data.OutputTokens, - Timestamp: data.Updated.UnixMilli(), + Model: s.Model, + Provider: s.Provider, + InputTokens: s.InputTokens, + OutputTokens: s.OutputTokens, + TotalTokens: s.InputTokens + s.OutputTokens, + Timestamp: s.Updated.UnixMilli(), }) } @@ -96,7 +96,8 @@ func (m *UsageMethods) handleGet(ctx context.Context, client *gateway.Client, re } func (m *UsageMethods) handleSummary(ctx context.Context, client *gateway.Client, req *protocol.RequestFrame) { - sessions := m.sessions.List(ctx, "") // all agents + // Use ListPagedRich: single query returns all token data — no N+1 GetOrCreate loop. + result := m.sessions.ListPagedRich(ctx, store.SessionListOpts{Limit: 10000}) type agentSummary struct { InputTokens int64 `json:"inputTokens"` @@ -108,9 +109,8 @@ func (m *UsageMethods) handleSummary(ctx context.Context, client *gateway.Client byAgent := make(map[string]*agentSummary) var totalRecords int - for _, s := range sessions { - data := m.sessions.GetOrCreate(ctx, s.Key) - if data.InputTokens == 0 && data.OutputTokens == 0 { + for _, s := range result.Sessions { + if s.InputTokens == 0 && s.OutputTokens == 0 { continue } @@ -119,9 +119,9 @@ func (m *UsageMethods) handleSummary(ctx context.Context, client *gateway.Client byAgent[agentID] = &agentSummary{} } - byAgent[agentID].InputTokens += data.InputTokens - byAgent[agentID].OutputTokens += data.OutputTokens - byAgent[agentID].TotalTokens += data.InputTokens + data.OutputTokens + byAgent[agentID].InputTokens += s.InputTokens + byAgent[agentID].OutputTokens += s.OutputTokens + byAgent[agentID].TotalTokens += s.InputTokens + s.OutputTokens byAgent[agentID].Sessions++ totalRecords++ } diff --git a/internal/http/agents.go b/internal/http/agents.go index b7bee08d..675332e7 100644 --- a/internal/http/agents.go +++ b/internal/http/agents.go @@ -18,6 +18,7 @@ import ( "github.com/nextlevelbuilder/goclaw/internal/permissions" "github.com/nextlevelbuilder/goclaw/internal/providers" "github.com/nextlevelbuilder/goclaw/internal/store" + "github.com/nextlevelbuilder/goclaw/internal/tools" "github.com/nextlevelbuilder/goclaw/pkg/protocol" ) @@ -32,8 +33,9 @@ type AgentsHandler struct { kgStore store.KnowledgeGraphStore // for import (nil = disabled) episodicStore store.EpisodicStore // for import (nil in SQLite/lite builds) vaultStore store.VaultStore // for vault import (nil = disabled) - toolsReg ToolLister // for system prompt preview tool resolution (nil = fallback) - skillsLoader SkillPinnedBuilder // for system prompt preview pinned skills (nil = skip) + toolsReg ToolPreviewLister // for system prompt preview tool resolution (nil = fallback) + skillsLoader SkillPreviewBuilder // for system prompt preview pinned skills (nil = skip) + skillAccessStore store.SkillAccessStore // for system prompt preview skill filtering (nil = skip) teamStore store.TeamStore // for system prompt preview team context (nil = skip) agentLinkStore store.AgentLinkStore // for system prompt preview delegation targets (nil = skip) defaultWorkspace string // default workspace path template (e.g. "~/.goclaw/workspace") @@ -82,24 +84,30 @@ func (h *AgentsHandler) SetVaultStore(vs store.VaultStore) { h.vaultStore = vs } -// ToolLister is satisfied by tools.Registry for system prompt preview. -type ToolLister interface{ List() []string } +// ToolPreviewLister is satisfied by tools.Registry for system prompt preview. +type ToolPreviewLister interface { + List() []string + Get(name string) (tools.Tool, bool) + Aliases() map[string]string +} -// SkillPinnedBuilder is satisfied by skills.Loader for pinned skills summary. -type SkillPinnedBuilder interface { +// SkillPreviewBuilder is satisfied by skills.Loader for system prompt preview. +type SkillPreviewBuilder interface { BuildPinnedSummary(ctx context.Context, names []string) string + BuildSummary(ctx context.Context, allowList []string) string } // SetPreviewDeps attaches optional dependencies for system prompt preview. -func (h *AgentsHandler) SetPreviewDeps(tl ToolLister, sl SkillPinnedBuilder) { +func (h *AgentsHandler) SetPreviewDeps(tl ToolPreviewLister, sl SkillPreviewBuilder) { h.toolsReg = tl h.skillsLoader = sl } // SetPreviewStores attaches team + agent link stores for system prompt preview. -func (h *AgentsHandler) SetPreviewStores(ts store.TeamStore, als store.AgentLinkStore) { +func (h *AgentsHandler) SetPreviewStores(ts store.TeamStore, als store.AgentLinkStore, sas store.SkillAccessStore) { h.teamStore = ts h.agentLinkStore = als + h.skillAccessStore = sas } // isOwnerUser checks if the given user ID is a system owner. diff --git a/internal/http/agents_import_sections.go b/internal/http/agents_import_sections.go index 91b303fe..bfc5cbe9 100644 --- a/internal/http/agents_import_sections.go +++ b/internal/http/agents_import_sections.go @@ -5,6 +5,7 @@ import ( "encoding/json" "fmt" "log/slog" + "strings" "github.com/google/uuid" "github.com/nextlevelbuilder/goclaw/internal/config" @@ -151,40 +152,42 @@ func (h *AgentsHandler) importKG(ctx context.Context, ag *store.AgentData, arc * func (h *AgentsHandler) importCron(ctx context.Context, ag *store.AgentData, arc *importArchive, summary *ImportSummary, progressFn func(ProgressEvent)) { tid := importTenantID(ctx) - for _, j := range arc.cronJobs { - // cron_jobs has no UNIQUE constraint on (agent_id, name), so use SELECT+UPDATE/INSERT - var exists bool - _ = h.db.QueryRowContext(ctx, - `SELECT EXISTS(SELECT 1 FROM cron_jobs WHERE agent_id = $1 AND name = $2 AND tenant_id = $3)`, - ag.ID, j.Name, tid, - ).Scan(&exists) - if exists { - _, err := h.db.ExecContext(ctx, - `UPDATE cron_jobs - SET schedule_kind=$1, cron_expression=$2, interval_ms=$3, - run_at=$4, timezone=$5, payload=$6, delete_after_run=$7 - WHERE agent_id=$8 AND name=$9 AND tenant_id=$10`, - j.ScheduleKind, j.CronExpression, j.IntervalMS, - nullStr(j.RunAt), j.Timezone, j.Payload, j.DeleteAfterRun, - ag.ID, j.Name, tid, - ) - if err != nil { - slog.Warn("agents.import.cron_job.update", "agent_id", ag.ID, "name", j.Name, "error", err) - } - } else { - _, err := h.db.ExecContext(ctx, - `INSERT INTO cron_jobs - (agent_id, name, enabled, schedule_kind, cron_expression, interval_ms, run_at, timezone, payload, delete_after_run, tenant_id) - VALUES ($1, $2, false, $3, $4, $5, $6, $7, $8, $9, $10)`, - ag.ID, j.Name, j.ScheduleKind, + const paramsPerRow = 10 // agent_id, name, schedule_kind, cron_expression, interval_ms, run_at, timezone, payload, delete_after_run, tenant_id (enabled is literal false) + const chunkSize = 5000 + + for start := 0; start < len(arc.cronJobs); start += chunkSize { + end := min(start+chunkSize, len(arc.cronJobs)) + chunk := arc.cronJobs[start:end] + + args := make([]any, 0, len(chunk)*paramsPerRow) + placeholders := make([]string, 0, len(chunk)) + for i, j := range chunk { + base := i * paramsPerRow + placeholders = append(placeholders, fmt.Sprintf( + "($%d,$%d,false,$%d,$%d,$%d,$%d,$%d,$%d,$%d,$%d)", + base+1, base+2, base+3, base+4, base+5, base+6, base+7, base+8, base+9, base+10, + )) + args = append(args, ag.ID, j.Name, j.ScheduleKind, j.CronExpression, j.IntervalMS, nullStr(j.RunAt), j.Timezone, j.Payload, j.DeleteAfterRun, tid, ) - if err != nil { - slog.Warn("agents.import.cron_job.insert", "agent_id", ag.ID, "name", j.Name, "error", err) - } } - summary.CronJobs++ + + q := `INSERT INTO cron_jobs + (agent_id, name, enabled, schedule_kind, cron_expression, interval_ms, run_at, timezone, payload, delete_after_run, tenant_id) + VALUES ` + strings.Join(placeholders, ",") + ` + ON CONFLICT (agent_id, tenant_id, name) DO UPDATE SET + schedule_kind = EXCLUDED.schedule_kind, + cron_expression = EXCLUDED.cron_expression, + interval_ms = EXCLUDED.interval_ms, + run_at = EXCLUDED.run_at, + timezone = EXCLUDED.timezone, + payload = EXCLUDED.payload, + delete_after_run = EXCLUDED.delete_after_run` + if _, err := h.db.ExecContext(ctx, q, args...); err != nil { + slog.Warn("agents.import.cron_jobs.batch", "agent_id", ag.ID, "count", len(chunk), "error", err) + } + summary.CronJobs += len(chunk) } if progressFn != nil { progressFn(ProgressEvent{Phase: "cron", Status: "done", Current: summary.CronJobs, Total: len(arc.cronJobs)}) @@ -194,18 +197,28 @@ func (h *AgentsHandler) importCron(ctx context.Context, ag *store.AgentData, arc func (h *AgentsHandler) importUserProfiles(ctx context.Context, ag *store.AgentData, arc *importArchive, summary *ImportSummary, progressFn func(ProgressEvent)) { // workspace=NULL for portability (auto-created via GetOrCreateUserProfile on first user access) tid := importTenantID(ctx) - for _, p := range arc.userProfiles { - _, err := h.db.ExecContext(ctx, - `INSERT INTO user_agent_profiles (agent_id, user_id, workspace, tenant_id) - VALUES ($1, $2, NULL, $3) - ON CONFLICT DO NOTHING`, - ag.ID, p.UserID, tid, - ) - if err != nil { - slog.Warn("agents.import.user_profile", "agent_id", ag.ID, "user_id", p.UserID, "error", err) - continue + const colsPerRow = 3 // agent_id, user_id, tenant_id + const chunkSize = 5000 + + for start := 0; start < len(arc.userProfiles); start += chunkSize { + end := min(start+chunkSize, len(arc.userProfiles)) + chunk := arc.userProfiles[start:end] + + args := make([]any, 0, len(chunk)*colsPerRow) + placeholders := make([]string, 0, len(chunk)) + for i, p := range chunk { + base := i * colsPerRow + placeholders = append(placeholders, fmt.Sprintf("($%d,$%d,NULL,$%d)", base+1, base+2, base+3)) + args = append(args, ag.ID, p.UserID, tid) } - summary.UserProfiles++ + + q := `INSERT INTO user_agent_profiles (agent_id, user_id, workspace, tenant_id) + VALUES ` + strings.Join(placeholders, ",") + ` + ON CONFLICT (agent_id, user_id) DO NOTHING` + if _, err := h.db.ExecContext(ctx, q, args...); err != nil { + slog.Warn("agents.import.user_profiles.batch", "agent_id", ag.ID, "count", len(chunk), "error", err) + } + summary.UserProfiles += len(chunk) } if progressFn != nil { progressFn(ProgressEvent{Phase: "user_profiles", Status: "done", Current: summary.UserProfiles, Total: len(arc.userProfiles)}) @@ -214,22 +227,35 @@ func (h *AgentsHandler) importUserProfiles(ctx context.Context, ag *store.AgentD func (h *AgentsHandler) importUserOverrides(ctx context.Context, ag *store.AgentData, arc *importArchive, summary *ImportSummary, progressFn func(ProgressEvent)) { tid := importTenantID(ctx) - for _, o := range arc.userOverrides { - _, err := h.db.ExecContext(ctx, - `INSERT INTO user_agent_overrides (agent_id, user_id, provider, model, settings, tenant_id) - VALUES ($1, $2, $3, $4, $5, $6) - ON CONFLICT (agent_id, user_id) DO UPDATE SET - provider = EXCLUDED.provider, - model = EXCLUDED.model, - settings = EXCLUDED.settings, - updated_at = NOW()`, - ag.ID, o.UserID, o.Provider, o.Model, coalesceJSON(o.Settings), tid, - ) - if err != nil { - slog.Warn("agents.import.user_override", "agent_id", ag.ID, "user_id", o.UserID, "error", err) - continue + const colsPerRow = 6 // agent_id, user_id, provider, model, settings, tenant_id + const chunkSize = 5000 + + for start := 0; start < len(arc.userOverrides); start += chunkSize { + end := min(start+chunkSize, len(arc.userOverrides)) + chunk := arc.userOverrides[start:end] + + args := make([]any, 0, len(chunk)*colsPerRow) + placeholders := make([]string, 0, len(chunk)) + for i, o := range chunk { + base := i * colsPerRow + placeholders = append(placeholders, fmt.Sprintf( + "($%d,$%d,$%d,$%d,$%d,$%d)", + base+1, base+2, base+3, base+4, base+5, base+6, + )) + args = append(args, ag.ID, o.UserID, o.Provider, o.Model, coalesceJSON(o.Settings), tid) } - summary.UserOverrides++ + + q := `INSERT INTO user_agent_overrides (agent_id, user_id, provider, model, settings, tenant_id) + VALUES ` + strings.Join(placeholders, ",") + ` + ON CONFLICT (agent_id, user_id) DO UPDATE SET + provider = EXCLUDED.provider, + model = EXCLUDED.model, + settings = EXCLUDED.settings, + updated_at = NOW()` + if _, err := h.db.ExecContext(ctx, q, args...); err != nil { + slog.Warn("agents.import.user_overrides.batch", "agent_id", ag.ID, "count", len(chunk), "error", err) + } + summary.UserOverrides += len(chunk) } if progressFn != nil { progressFn(ProgressEvent{Phase: "user_overrides", Status: "done", Current: summary.UserOverrides, Total: len(arc.userOverrides)}) @@ -354,9 +380,10 @@ func (h *AgentsHandler) importVault(ctx context.Context, ag *store.AgentData, ar progressFn(ProgressEvent{Phase: "vault_documents", Status: "running", Total: len(arc.vaultDocuments)}) } for _, d := range arc.vaultDocuments { + agentIDStr := ag.ID.String() doc := &store.VaultDocument{ TenantID: tid.String(), - AgentID: ag.ID.String(), + AgentID: &agentIDStr, TeamID: nil, // team_id not portable Scope: d.Scope, CustomScope: d.CustomScope, diff --git a/internal/http/agents_import_team.go b/internal/http/agents_import_team.go index 0e246261..9d0a3199 100644 --- a/internal/http/agents_import_team.go +++ b/internal/http/agents_import_team.go @@ -5,6 +5,7 @@ import ( "fmt" "log/slog" "path/filepath" + "strings" "github.com/google/uuid" @@ -60,20 +61,37 @@ func (h *AgentsHandler) importTeamSection(ctx context.Context, ag *store.AgentDa // Always include the importing agent itself agentKeyToID[ag.AgentKey] = ag.ID - // Insert non-lead members - for _, m := range arc.teamMembers { - memberID, ok := agentKeyToID[m.AgentKey] - if !ok { - slog.Info("import.team: member not found, skipping", "agent_key", m.AgentKey) - continue + // Insert non-lead members (batch) + { + type memberRow struct { + agentID uuid.UUID + role string } - if _, err = h.db.ExecContext(ctx, - `INSERT INTO agent_team_members (team_id, agent_id, role, tenant_id, joined_at) - VALUES ($1, $2, $3, $4, NOW()) - ON CONFLICT (team_id, agent_id) DO NOTHING`, - teamID, memberID, m.Role, tid, - ); err != nil { - slog.Warn("import.team: insert member", "agent_key", m.AgentKey, "error", err) + var rows []memberRow + for _, m := range arc.teamMembers { + memberID, ok := agentKeyToID[m.AgentKey] + if !ok { + slog.Info("import.team: member not found, skipping", "agent_key", m.AgentKey) + continue + } + rows = append(rows, memberRow{agentID: memberID, role: m.Role}) + } + const cols = 4 // team_id, agent_id, role, tenant_id + for start := 0; start < len(rows); start += 1000 { + end := min(start+1000, len(rows)) + chunk := rows[start:end] + args := make([]any, 0, len(chunk)*cols) + ph := make([]string, 0, len(chunk)) + for i, r := range chunk { + b := i * cols + ph = append(ph, fmt.Sprintf("($%d,$%d,$%d,$%d,NOW())", b+1, b+2, b+3, b+4)) + args = append(args, teamID, r.agentID, r.role, tid) + } + q := `INSERT INTO agent_team_members (team_id, agent_id, role, tenant_id, joined_at) + VALUES ` + strings.Join(ph, ",") + ` ON CONFLICT (team_id, agent_id) DO NOTHING` + if _, err = h.db.ExecContext(ctx, q, args...); err != nil { + slog.Warn("import.team: batch insert members", "count", len(chunk), "error", err) + } } } @@ -130,62 +148,132 @@ func (h *AgentsHandler) importTeamSection(ctx context.Context, ag *store.AgentDa } } - // Insert comments - for _, c := range arc.teamComments { - if c.TaskIdx < 0 || c.TaskIdx >= len(taskIDByIdx) { - continue + // Insert comments (batch) + { + type commentRow struct { + id uuid.UUID + taskID uuid.UUID + agentID *uuid.UUID + userID *string + content string + commentType string + metadata any } - taskID := taskIDByIdx[c.TaskIdx] - - var agentID *uuid.UUID - if c.AgentKey != nil { - if id, ok := agentKeyToID[*c.AgentKey]; ok { - agentID = &id + var cRows []commentRow + for _, c := range arc.teamComments { + if c.TaskIdx < 0 || c.TaskIdx >= len(taskIDByIdx) { + continue + } + var agentID *uuid.UUID + if c.AgentKey != nil { + if id, ok := agentKeyToID[*c.AgentKey]; ok { + agentID = &id + } + } + cRows = append(cRows, commentRow{ + id: uuid.Must(uuid.NewV7()), taskID: taskIDByIdx[c.TaskIdx], + agentID: agentID, userID: c.UserID, content: c.Content, + commentType: c.CommentType, metadata: nullJSON(c.Metadata), + }) + } + const cols = 8 + for start := 0; start < len(cRows); start += 1000 { + end := min(start+1000, len(cRows)) + chunk := cRows[start:end] + args := make([]any, 0, len(chunk)*cols) + ph := make([]string, 0, len(chunk)) + for i, r := range chunk { + b := i * cols + ph = append(ph, fmt.Sprintf("($%d,$%d,$%d,$%d,$%d,$%d,$%d,$%d,NOW())", b+1, b+2, b+3, b+4, b+5, b+6, b+7, b+8)) + args = append(args, r.id, r.taskID, r.agentID, r.userID, r.content, r.commentType, r.metadata, tid) + } + q := `INSERT INTO team_task_comments (id, task_id, agent_id, user_id, content, comment_type, metadata, tenant_id, created_at) + VALUES ` + strings.Join(ph, ",") + if _, err = h.db.ExecContext(ctx, q, args...); err != nil { + slog.Warn("import.team: batch insert comments", "count", len(chunk), "error", err) } } - if _, err = h.db.ExecContext(ctx, - `INSERT INTO team_task_comments (id, task_id, agent_id, user_id, content, comment_type, metadata, tenant_id, created_at) - VALUES ($1,$2,$3,$4,$5,$6,$7,$8,NOW())`, - uuid.Must(uuid.NewV7()), taskID, agentID, c.UserID, - c.Content, c.CommentType, nullJSON(c.Metadata), tid, - ); err != nil { - slog.Warn("import.team: insert comment", "task_id", taskID, "error", err) + } + + // Insert events (batch) + { + type eventRow struct { + id uuid.UUID + taskID uuid.UUID + eventType string + actorType string + actorID string + data any + } + var eRows []eventRow + for _, ev := range arc.teamEvents { + if ev.TaskIdx < 0 || ev.TaskIdx >= len(taskIDByIdx) { + continue + } + eRows = append(eRows, eventRow{ + id: uuid.Must(uuid.NewV7()), taskID: taskIDByIdx[ev.TaskIdx], + eventType: ev.EventType, actorType: ev.ActorType, + actorID: ev.ActorID, data: nullJSON(ev.Data), + }) + } + const cols = 7 + for start := 0; start < len(eRows); start += 1000 { + end := min(start+1000, len(eRows)) + chunk := eRows[start:end] + args := make([]any, 0, len(chunk)*cols) + ph := make([]string, 0, len(chunk)) + for i, r := range chunk { + b := i * cols + ph = append(ph, fmt.Sprintf("($%d,$%d,$%d,$%d,$%d,$%d,$%d,NOW())", b+1, b+2, b+3, b+4, b+5, b+6, b+7)) + args = append(args, r.id, r.taskID, r.eventType, r.actorType, r.actorID, r.data, tid) + } + q := `INSERT INTO team_task_events (id, task_id, event_type, actor_type, actor_id, data, tenant_id, created_at) + VALUES ` + strings.Join(ph, ",") + if _, err = h.db.ExecContext(ctx, q, args...); err != nil { + slog.Warn("import.team: batch insert events", "count", len(chunk), "error", err) + } } } - // Insert events - for _, ev := range arc.teamEvents { - if ev.TaskIdx < 0 || ev.TaskIdx >= len(taskIDByIdx) { - continue + // Insert agent_links (batch) + { + type linkRow struct { + id uuid.UUID + srcID uuid.UUID + tgtID uuid.UUID + dir string + desc string } - taskID := taskIDByIdx[ev.TaskIdx] - if _, err = h.db.ExecContext(ctx, - `INSERT INTO team_task_events (id, task_id, event_type, actor_type, actor_id, data, tenant_id, created_at) - VALUES ($1,$2,$3,$4,$5,$6,$7,NOW())`, - uuid.Must(uuid.NewV7()), taskID, - ev.EventType, ev.ActorType, ev.ActorID, nullJSON(ev.Data), tid, - ); err != nil { - slog.Warn("import.team: insert event", "task_id", taskID, "error", err) + var lRows []linkRow + for _, l := range arc.teamLinks { + srcID, srcOK := agentKeyToID[l.SourceAgentKey] + tgtID, tgtOK := agentKeyToID[l.TargetAgentKey] + if !srcOK || !tgtOK { + slog.Info("import.team: agent_link skipped — agent not found", + "source", l.SourceAgentKey, "target", l.TargetAgentKey) + continue + } + lRows = append(lRows, linkRow{ + id: uuid.Must(uuid.NewV7()), srcID: srcID, tgtID: tgtID, + dir: l.Direction, desc: l.Description, + }) } - } - - // Insert agent_links - for _, l := range arc.teamLinks { - srcID, srcOK := agentKeyToID[l.SourceAgentKey] - tgtID, tgtOK := agentKeyToID[l.TargetAgentKey] - if !srcOK || !tgtOK { - slog.Info("import.team: agent_link skipped — agent not found", - "source", l.SourceAgentKey, "target", l.TargetAgentKey) - continue - } - if _, err = h.db.ExecContext(ctx, - `INSERT INTO agent_links (id, source_agent_id, target_agent_id, direction, description, created_by, tenant_id) - VALUES ($1,$2,$3,$4,$5,$6,$7) - ON CONFLICT DO NOTHING`, - uuid.Must(uuid.NewV7()), srcID, tgtID, - l.Direction, l.Description, userID, tid, - ); err != nil { - slog.Warn("import.team: insert link", "error", err) + const cols = 7 + for start := 0; start < len(lRows); start += 1000 { + end := min(start+1000, len(lRows)) + chunk := lRows[start:end] + args := make([]any, 0, len(chunk)*cols) + ph := make([]string, 0, len(chunk)) + for i, r := range chunk { + b := i * cols + ph = append(ph, fmt.Sprintf("($%d,$%d,$%d,$%d,$%d,$%d,$%d)", b+1, b+2, b+3, b+4, b+5, b+6, b+7)) + args = append(args, r.id, r.srcID, r.tgtID, r.dir, r.desc, userID, tid) + } + q := `INSERT INTO agent_links (id, source_agent_id, target_agent_id, direction, description, created_by, tenant_id) + VALUES ` + strings.Join(ph, ",") + ` ON CONFLICT DO NOTHING` + if _, err = h.db.ExecContext(ctx, q, args...); err != nil { + slog.Warn("import.team: batch insert links", "count", len(chunk), "error", err) + } } } @@ -208,6 +296,7 @@ func (h *AgentsHandler) importTeamSection(ctx context.Context, ag *store.AgentDa } // buildAgentKeyMap resolves all agent_keys referenced in the team section to UUIDs. +// Uses batch GetByKeys with tenant-scoped context instead of per-key SELECT. func (h *AgentsHandler) buildAgentKeyMap(ctx context.Context, tid uuid.UUID, arc *importArchive) map[string]uuid.UUID { keys := make(map[string]struct{}) for _, m := range arc.teamMembers { @@ -231,14 +320,22 @@ func (h *AgentsHandler) buildAgentKeyMap(ctx context.Context, tid uuid.UUID, arc keys[l.TargetAgentKey] = struct{}{} } - out := make(map[string]uuid.UUID, len(keys)) - for key := range keys { - var id uuid.UUID - if err := h.db.QueryRowContext(ctx, - `SELECT id FROM agents WHERE agent_key = $1 AND tenant_id = $2`, key, tid, - ).Scan(&id); err == nil { - out[key] = id - } + keyList := make([]string, 0, len(keys)) + for k := range keys { + keyList = append(keyList, k) + } + + // Ensure tenant-scoped context for GetByKeys (Red Team Fix #2). + scopedCtx := store.WithTenantID(ctx, tid) + agents, err := h.agents.GetByKeys(scopedCtx, keyList) + if err != nil { + slog.Warn("import.team: batch agent key lookup failed", "error", err) + return make(map[string]uuid.UUID) + } + + out := make(map[string]uuid.UUID, len(agents)) + for _, a := range agents { + out[a.AgentKey] = a.ID } return out } diff --git a/internal/http/agents_prompt_preview.go b/internal/http/agents_prompt_preview.go index 95c0b4e3..c5a04526 100644 --- a/internal/http/agents_prompt_preview.go +++ b/internal/http/agents_prompt_preview.go @@ -6,6 +6,7 @@ import ( "strings" "github.com/nextlevelbuilder/goclaw/internal/agent" + "github.com/nextlevelbuilder/goclaw/internal/providers" "github.com/nextlevelbuilder/goclaw/internal/tokencount" ) @@ -18,10 +19,11 @@ type promptPreviewSection struct { // promptPreviewResponse is the API response for system prompt preview. type promptPreviewResponse struct { - Mode string `json:"mode"` - Prompt string `json:"prompt"` - TokenCount int `json:"token_count"` - Sections []promptPreviewSection `json:"sections"` + Mode string `json:"mode"` + Prompt string `json:"prompt"` + TokenCount int `json:"token_count"` + Sections []promptPreviewSection `json:"sections"` + Tools []providers.ToolDefinition `json:"tools,omitempty"` } // handleSystemPromptPreview renders the actual system prompt for an agent in a given mode. @@ -49,26 +51,28 @@ func (h *AgentsHandler) handleSystemPromptPreview(w http.ResponseWriter, r *http // Build preview prompt — reuses same BuildSystemPrompt() as LLM pipeline. // Runtime-only fields (channel, peer kind, credentials) are zero-valued; // BuildSystemPrompt nil-checks every field so these sections are simply skipped. - prompt := agent.BuildPreviewPrompt(ctx, ag, mode, r.URL.Query().Get("user_id"), agent.PreviewDeps{ - AgentStore: h.agents, - TeamStore: h.teamStore, - AgentLinks: h.agentLinkStore, - ProviderReg: h.providerReg, - ToolLister: h.toolsReg, - SkillsLoader: h.skillsLoader, - DataDir: h.dataDir, + result := agent.BuildPreviewPrompt(ctx, ag, mode, r.URL.Query().Get("user_id"), agent.PreviewDeps{ + AgentStore: h.agents, + TeamStore: h.teamStore, + AgentLinks: h.agentLinkStore, + ProviderReg: h.providerReg, + ToolLister: h.toolsReg, + SkillsLoader: h.skillsLoader, + SkillAccessStore: h.skillAccessStore, + DataDir: h.dataDir, }) counter := tokencount.NewFallbackCounter() - tokens := counter.Count("claude-3", prompt) - sections := parseSections(prompt) + tokens := counter.Count("claude-3", result.Prompt) + sections := parseSections(result.Prompt) w.Header().Set("Content-Type", "application/json") json.NewEncoder(w).Encode(promptPreviewResponse{ Mode: string(mode), - Prompt: prompt, + Prompt: result.Prompt, TokenCount: tokens, Sections: sections, + Tools: result.ToolDefs, }) } diff --git a/internal/http/auth_test.go b/internal/http/auth_test.go index c5c07552..bc2869e8 100644 --- a/internal/http/auth_test.go +++ b/internal/http/auth_test.go @@ -117,6 +117,9 @@ func (m *mockTenantStore) GetTenantUser(context.Context, uuid.UUID) (*store.Tena func (m *mockTenantStore) CreateTenantUserReturning(context.Context, uuid.UUID, string, string, string) (*store.TenantUserData, error) { return nil, fmt.Errorf("not implemented") } +func (m *mockTenantStore) GetTenantsByIDs(context.Context, []uuid.UUID) ([]store.TenantData, error) { + return nil, nil +} type mockPairingStore struct { paired map[string]bool diff --git a/internal/http/channel_instances.go b/internal/http/channel_instances.go index d762501d..f5529a8b 100644 --- a/internal/http/channel_instances.go +++ b/internal/http/channel_instances.go @@ -540,7 +540,7 @@ func (h *ChannelInstancesHandler) handleResolveContacts(w http.ResponseWriter, r // isValidChannelType checks if the channel type is supported. func isValidChannelType(ct string) bool { switch ct { - case "telegram", "discord", "slack", "whatsapp", "zalo_oa", "zalo_personal", "feishu": + case "telegram", "discord", "slack", "whatsapp", "zalo_oa", "zalo_personal", "feishu", "facebook", "pancake": return true } return false diff --git a/internal/http/evolution_handlers.go b/internal/http/evolution_handlers.go index 87572ca8..7996952e 100644 --- a/internal/http/evolution_handlers.go +++ b/internal/http/evolution_handlers.go @@ -1,6 +1,7 @@ package http import ( + "encoding/json" "log/slog" "net/http" "strconv" @@ -26,6 +27,9 @@ type EvolutionHandler struct { // Optional: agent store for applying threshold suggestions. agentStore store.AgentStore + + // Optional: tenant tool config for disabling tools on SuggestToolOrder approval. + toolTenantCfgs store.BuiltinToolTenantConfigStore } // EvolutionHandlerOpt configures optional EvolutionHandler dependencies. @@ -45,6 +49,11 @@ func WithAgentStore(as store.AgentStore) EvolutionHandlerOpt { return func(h *EvolutionHandler) { h.agentStore = as } } +// WithToolTenantCfgs enables tool disabling on SuggestToolOrder approval. +func WithToolTenantCfgs(tc store.BuiltinToolTenantConfigStore) EvolutionHandlerOpt { + return func(h *EvolutionHandler) { h.toolTenantCfgs = tc } +} + func NewEvolutionHandler(m store.EvolutionMetricsStore, s store.EvolutionSuggestionStore, opts ...EvolutionHandlerOpt) *EvolutionHandler { h := &EvolutionHandler{metrics: m, suggestions: s} for _, opt := range opts { @@ -222,6 +231,29 @@ func (h *EvolutionHandler) handleUpdateSuggestion(w http.ResponseWriter, r *http writeJSON(w, http.StatusOK, map[string]string{"status": "ok", "action": "skill_created"}) return + case store.SuggestToolOrder: + action := "tool_order_approved" + if h.toolTenantCfgs != nil { + // Extract tool name from suggestion parameters. + var params map[string]any + if err := json.Unmarshal(existing.Parameters, ¶ms); err == nil { + if toolName, _ := params["tool"].(string); toolName != "" { + // Disable tool at tenant level using existing infrastructure. + if err := h.toolTenantCfgs.Set(r.Context(), existing.TenantID, toolName, false); err != nil { + slog.Warn("evolution.tool_order.disable_failed", "tool", toolName, "error", err) + } else { + action = "tool_disabled" + } + } + } + } + if err := h.suggestions.UpdateSuggestionStatus(r.Context(), suggestionID, "applied", reviewedBy); err != nil { + writeJSON(w, http.StatusInternalServerError, map[string]string{"error": err.Error()}) + return + } + writeJSON(w, http.StatusOK, map[string]string{"status": "ok", "action": action}) + return + case store.SuggestThreshold: if h.agentStore == nil { writeJSON(w, http.StatusBadRequest, map[string]string{"error": "threshold auto-apply not available"}) @@ -240,7 +272,7 @@ func (h *EvolutionHandler) handleUpdateSuggestion(w http.ResponseWriter, r *http writeJSON(w, http.StatusBadRequest, map[string]string{"error": err.Error()}) return } - if err := agent.ApplySuggestion(r.Context(), h.agentStore, h.suggestions, *existing); err != nil { + if err := agent.ApplySuggestion(r.Context(), h.agentStore, h.suggestions, *existing, guardrails); err != nil { writeJSON(w, http.StatusInternalServerError, map[string]string{"error": err.Error()}) return } diff --git a/internal/http/vault_handler_documents.go b/internal/http/vault_handler_documents.go new file mode 100644 index 00000000..1a589f1e --- /dev/null +++ b/internal/http/vault_handler_documents.go @@ -0,0 +1,273 @@ +package http + +import ( + "log/slog" + "net/http" + "strings" + + "github.com/nextlevelbuilder/goclaw/internal/store" +) + +// handleListAllDocuments lists vault documents across all agents in tenant. +// Optional query param agent_id to filter by specific agent. +func (h *VaultHandler) handleListAllDocuments(w http.ResponseWriter, r *http.Request) { + tenantID := store.TenantIDFromContext(r.Context()) + agentID := r.URL.Query().Get("agent_id") + opts := h.parseListOpts(r) + + // Validate team membership if specific team requested. + if opts.TeamID != nil && *opts.TeamID != "" { + if !h.validateTeamMembership(r.Context(), w, *opts.TeamID) { + return + } + } + // Non-owner without team_id filter: show personal + user's teams. + if opts.TeamID == nil && !store.IsOwnerRole(r.Context()) { + h.applyNonOwnerTeamScope(r.Context(), &opts) + } + + docs, err := h.store.ListDocuments(r.Context(), tenantID.String(), agentID, opts) + if err != nil { + slog.Warn("vault.list_all failed", "error", err) + writeJSON(w, http.StatusInternalServerError, map[string]string{"error": err.Error()}) + return + } + if docs == nil { + docs = []store.VaultDocument{} + } + total, cntErr := h.store.CountDocuments(r.Context(), tenantID.String(), agentID, opts) + if cntErr != nil { + slog.Warn("vault.count failed", "error", cntErr) + } + writeJSON(w, http.StatusOK, vaultDocListResponse{Documents: docs, Total: total}) +} + +// handleListDocuments lists vault documents for a specific agent. +func (h *VaultHandler) handleListDocuments(w http.ResponseWriter, r *http.Request) { + tenantID := store.TenantIDFromContext(r.Context()) + agentID := r.PathValue("agentID") + opts := h.parseListOpts(r) + + if opts.TeamID != nil && *opts.TeamID != "" { + if !h.validateTeamMembership(r.Context(), w, *opts.TeamID) { + return + } + } + if opts.TeamID == nil && !store.IsOwnerRole(r.Context()) { + h.applyNonOwnerTeamScope(r.Context(), &opts) + } + + docs, err := h.store.ListDocuments(r.Context(), tenantID.String(), agentID, opts) + if err != nil { + slog.Warn("vault.list failed", "error", err) + writeJSON(w, http.StatusInternalServerError, map[string]string{"error": err.Error()}) + return + } + if docs == nil { + docs = []store.VaultDocument{} + } + total, cntErr := h.store.CountDocuments(r.Context(), tenantID.String(), agentID, opts) + if cntErr != nil { + slog.Warn("vault.count failed", "error", cntErr) + } + writeJSON(w, http.StatusOK, vaultDocListResponse{Documents: docs, Total: total}) +} + +// handleGetDocument returns a single vault document by ID, scoped to the agent. +func (h *VaultHandler) handleGetDocument(w http.ResponseWriter, r *http.Request) { + tenantID := store.TenantIDFromContext(r.Context()) + agentID := r.PathValue("agentID") + docID := r.PathValue("docID") + + doc, err := h.store.GetDocumentByID(r.Context(), tenantID.String(), docID) + if err != nil { + slog.Warn("vault.get failed", "error", err) + writeJSON(w, http.StatusInternalServerError, map[string]string{"error": err.Error()}) + return + } + if doc == nil || (agentID != "" && doc.AgentID != nil && *doc.AgentID != agentID) { + writeJSON(w, http.StatusNotFound, map[string]string{"error": "document not found"}) + return + } + // Verify team boundary — non-owner must be team member to view team docs. + if doc.TeamID != nil && *doc.TeamID != "" && !store.IsOwnerRole(r.Context()) { + if !h.validateTeamMembership(r.Context(), w, *doc.TeamID) { + return + } + } + writeJSON(w, http.StatusOK, doc) +} + +// handleCreateDocument creates a new vault document. +func (h *VaultHandler) handleCreateDocument(w http.ResponseWriter, r *http.Request) { + locale := extractLocale(r) + tenantID := store.TenantIDFromContext(r.Context()) + agentID := r.PathValue("agentID") + + var body struct { + Path string `json:"path"` + Title string `json:"title"` + DocType string `json:"doc_type"` + Scope string `json:"scope"` + TeamID string `json:"team_id"` + Metadata map[string]any `json:"metadata"` + } + if !bindJSON(w, r, locale, &body) { + return + } + if body.Path == "" || body.Title == "" { + writeJSON(w, http.StatusBadRequest, map[string]string{"error": "path and title are required"}) + return + } + if strings.Contains(body.Path, "..") || strings.HasPrefix(body.Path, "/") { + writeJSON(w, http.StatusBadRequest, map[string]string{"error": "invalid path"}) + return + } + if body.DocType == "" { + body.DocType = "note" + } + if body.Scope == "" { + body.Scope = "personal" + } + if !validDocType(body.DocType) { + writeJSON(w, http.StatusBadRequest, map[string]string{"error": "invalid doc_type"}) + return + } + if !validScope(body.Scope) { + writeJSON(w, http.StatusBadRequest, map[string]string{"error": "invalid scope"}) + return + } + + doc := &store.VaultDocument{ + TenantID: tenantID.String(), + Path: body.Path, + Title: body.Title, + DocType: body.DocType, + Scope: body.Scope, + Metadata: body.Metadata, + } + if agentID != "" { + doc.AgentID = &agentID + } else if doc.Scope == "personal" { + doc.Scope = "shared" // no agent → shared scope + } + if body.TeamID != "" { + if !h.validateTeamMembership(r.Context(), w, body.TeamID) { + return + } + doc.TeamID = &body.TeamID + if body.Scope == "personal" { + doc.Scope = "team" + } + } + if err := h.store.UpsertDocument(r.Context(), doc); err != nil { + slog.Warn("vault.create failed", "error", err) + writeJSON(w, http.StatusInternalServerError, map[string]string{"error": err.Error()}) + return + } + // Re-fetch by ID (set via RETURNING) — unambiguous even when same path exists across teams. + created, _ := h.store.GetDocumentByID(r.Context(), tenantID.String(), doc.ID) + if created != nil { + writeJSON(w, http.StatusCreated, created) + } else { + writeJSON(w, http.StatusCreated, doc) + } +} + +// handleUpdateDocument updates an existing vault document. +func (h *VaultHandler) handleUpdateDocument(w http.ResponseWriter, r *http.Request) { + locale := extractLocale(r) + tenantID := store.TenantIDFromContext(r.Context()) + agentID := r.PathValue("agentID") + docID := r.PathValue("docID") + + existing, err := h.store.GetDocumentByID(r.Context(), tenantID.String(), docID) + if err != nil || existing == nil || (agentID != "" && existing.AgentID != nil && *existing.AgentID != agentID) { + writeJSON(w, http.StatusNotFound, map[string]string{"error": "document not found"}) + return + } + + var body struct { + Title *string `json:"title"` + DocType *string `json:"doc_type"` + Scope *string `json:"scope"` + TeamID *string `json:"team_id"` // nil=no change, ""=clear, "uuid"=set + Metadata map[string]any `json:"metadata"` + } + if !bindJSON(w, r, locale, &body) { + return + } + + if body.Title != nil { + existing.Title = *body.Title + } + if body.DocType != nil { + existing.DocType = *body.DocType + } + if body.Scope != nil { + existing.Scope = *body.Scope + } + if body.TeamID != nil { + // Only owner/admin can change team assignment. + if !store.IsOwnerRole(r.Context()) { + writeJSON(w, http.StatusForbidden, map[string]string{"error": "only owner can change document team assignment"}) + return + } + if *body.TeamID == "" { + existing.TeamID = nil + existing.Scope = "personal" + } else { + existing.TeamID = body.TeamID + existing.Scope = "team" + } + } + if body.Metadata != nil { + existing.Metadata = body.Metadata + } + + if err := h.store.UpsertDocument(r.Context(), existing); err != nil { + slog.Warn("vault.update failed", "error", err) + writeJSON(w, http.StatusInternalServerError, map[string]string{"error": err.Error()}) + return + } + updated, _ := h.store.GetDocumentByID(r.Context(), tenantID.String(), docID) + if updated != nil { + writeJSON(w, http.StatusOK, updated) + } else { + writeJSON(w, http.StatusOK, existing) + } +} + +// handleDeleteDocument deletes a vault document by ID. +func (h *VaultHandler) handleDeleteDocument(w http.ResponseWriter, r *http.Request) { + tenantID := store.TenantIDFromContext(r.Context()) + agentID := r.PathValue("agentID") + docID := r.PathValue("docID") + + existing, err := h.store.GetDocumentByID(r.Context(), tenantID.String(), docID) + if err != nil || existing == nil || (agentID != "" && existing.AgentID != nil && *existing.AgentID != agentID) { + writeJSON(w, http.StatusNotFound, map[string]string{"error": "document not found"}) + return + } + + // Verify team boundary before deletion. + if existing.TeamID != nil && *existing.TeamID != "" && !store.IsOwnerRole(r.Context()) { + if !h.validateTeamMembership(r.Context(), w, *existing.TeamID) { + return + } + } + + // DeleteDocument without RunContext applies no team_id filter (broad match on tenant+agent+path). + // This is safe because we pre-validated team membership above and use server-derived existing.Path. + // Use the doc's actual agent_id (may be empty for team/shared docs). + deleteAgentID := "" + if existing.AgentID != nil { + deleteAgentID = *existing.AgentID + } + if err := h.store.DeleteDocument(r.Context(), tenantID.String(), deleteAgentID, existing.Path); err != nil { + slog.Warn("vault.delete failed", "error", err) + writeJSON(w, http.StatusInternalServerError, map[string]string{"error": err.Error()}) + return + } + w.WriteHeader(http.StatusNoContent) +} diff --git a/internal/http/vault_handler_links.go b/internal/http/vault_handler_links.go new file mode 100644 index 00000000..1ddf0bf3 --- /dev/null +++ b/internal/http/vault_handler_links.go @@ -0,0 +1,255 @@ +package http + +import ( + "log/slog" + "net/http" + "strings" + + "github.com/nextlevelbuilder/goclaw/internal/store" +) + +// handleSearchAll runs tenant-wide search (agent_id optional in body). +func (h *VaultHandler) handleSearchAll(w http.ResponseWriter, r *http.Request) { + h.doSearch(w, r, "") +} + +// handleSearch runs hybrid FTS+vector search scoped to a specific agent. +func (h *VaultHandler) handleSearch(w http.ResponseWriter, r *http.Request) { + h.doSearch(w, r, r.PathValue("agentID")) +} + +// doSearch is the shared search implementation for both per-agent and tenant-wide endpoints. +func (h *VaultHandler) doSearch(w http.ResponseWriter, r *http.Request, agentID string) { + locale := extractLocale(r) + tenantID := store.TenantIDFromContext(r.Context()) + + var body struct { + Query string `json:"query"` + AgentID string `json:"agent_id"` + Scope string `json:"scope"` + DocTypes []string `json:"doc_types"` + MaxResults int `json:"max_results"` + TeamID string `json:"team_id"` + } + if !bindJSON(w, r, locale, &body) { + return + } + if body.Query == "" { + writeJSON(w, http.StatusBadRequest, map[string]string{"error": "query is required"}) + return + } + if body.MaxResults <= 0 { + body.MaxResults = 10 + } + // Body agent_id only used when path doesn't provide one (tenant-wide endpoint). + if agentID == "" { + agentID = body.AgentID + } + + searchOpts := store.VaultSearchOptions{ + Query: body.Query, + AgentID: agentID, + TenantID: tenantID.String(), + Scope: body.Scope, + DocTypes: body.DocTypes, + MaxResults: body.MaxResults, + } + if body.TeamID != "" { + if !h.validateTeamMembership(r.Context(), w, body.TeamID) { + return + } + searchOpts.TeamID = &body.TeamID + } else if !store.IsOwnerRole(r.Context()) { + if ids := h.userAccessibleTeamIDs(r.Context()); len(ids) > 0 { + searchOpts.TeamIDs = ids + } else { + empty := "" + searchOpts.TeamID = &empty + } + } + + results, err := h.store.Search(r.Context(), searchOpts) + if err != nil { + slog.Warn("vault.search failed", "error", err) + writeJSON(w, http.StatusInternalServerError, map[string]string{"error": err.Error()}) + return + } + if results == nil { + results = []store.VaultSearchResult{} + } + writeJSON(w, http.StatusOK, results) +} + +// handleGetLinks returns outgoing links and backlinks for a vault document. +func (h *VaultHandler) handleGetLinks(w http.ResponseWriter, r *http.Request) { + tenantID := store.TenantIDFromContext(r.Context()) + _ = r.PathValue("agentID") // agent scoping done at document level + docID := r.PathValue("docID") + + outLinks, err := h.store.GetOutLinks(r.Context(), tenantID.String(), docID) + if err != nil { + slog.Warn("vault.outlinks failed", "error", err) + writeJSON(w, http.StatusInternalServerError, map[string]string{"error": err.Error()}) + return + } + backlinks, err := h.store.GetBacklinks(r.Context(), tenantID.String(), docID) + if err != nil { + slog.Warn("vault.backlinks failed", "error", err) + writeJSON(w, http.StatusInternalServerError, map[string]string{"error": err.Error()}) + return + } + if outLinks == nil { + outLinks = []store.VaultLink{} + } + if backlinks == nil { + backlinks = []store.VaultBacklink{} + } + + // Filter backlinks by team boundary — derive team context from the target document + // itself (not a query param) so clients don't need to supply it correctly. + isOwner := store.IsOwnerRole(r.Context()) + if !isOwner { + targetDoc, _ := h.store.GetDocumentByID(r.Context(), tenantID.String(), docID) + var currentTeamID string + if targetDoc != nil && targetDoc.TeamID != nil { + currentTeamID = *targetDoc.TeamID + } + filtered := make([]store.VaultBacklink, 0, len(backlinks)) + for _, bl := range backlinks { + if currentTeamID != "" { + if bl.TeamID != nil && *bl.TeamID != currentTeamID { + continue + } + } else { + if bl.TeamID != nil && *bl.TeamID != "" { + continue + } + } + filtered = append(filtered, bl) + } + backlinks = filtered + } + + // Resolve target doc titles for outlinks so the UI can display names instead of IDs. + docNames := make(map[string]string, len(outLinks)) + for _, l := range outLinks { + if _, seen := docNames[l.ToDocID]; seen { + continue + } + if d, err := h.store.GetDocumentByID(r.Context(), tenantID.String(), l.ToDocID); err == nil && d != nil { + name := d.Title + if name == "" { + if idx := strings.LastIndex(d.Path, "/"); idx >= 0 { + name = d.Path[idx+1:] + } else { + name = d.Path + } + } + docNames[l.ToDocID] = name + } + } + + writeJSON(w, http.StatusOK, map[string]any{ + "outlinks": outLinks, + "backlinks": backlinks, + "doc_names": docNames, + }) +} + +// handleCreateLink creates a link between two vault documents. +func (h *VaultHandler) handleCreateLink(w http.ResponseWriter, r *http.Request) { + locale := extractLocale(r) + tenantID := store.TenantIDFromContext(r.Context()) + + var body struct { + FromDocID string `json:"from_doc_id"` + ToDocID string `json:"to_doc_id"` + LinkType string `json:"link_type"` + Context string `json:"context"` + } + if !bindJSON(w, r, locale, &body) { + return + } + if body.FromDocID == "" || body.ToDocID == "" { + writeJSON(w, http.StatusBadRequest, map[string]string{"error": "from_doc_id and to_doc_id are required"}) + return + } + if body.LinkType == "" { + body.LinkType = "reference" + } + + // Verify both docs exist, same tenant, and at least source belongs to this agent. + agentID := r.PathValue("agentID") + from, _ := h.store.GetDocumentByID(r.Context(), tenantID.String(), body.FromDocID) + to, _ := h.store.GetDocumentByID(r.Context(), tenantID.String(), body.ToDocID) + if from == nil || to == nil { + writeJSON(w, http.StatusNotFound, map[string]string{"error": "one or both documents not found"}) + return + } + if agentID != "" && from.AgentID != nil && *from.AgentID != agentID { + writeJSON(w, http.StatusForbidden, map[string]string{"error": "source document does not belong to this agent"}) + return + } + // Block cross-team linking (both team docs must be in same team). + if from.TeamID != nil && to.TeamID != nil && *from.TeamID != *to.TeamID { + writeJSON(w, http.StatusBadRequest, map[string]string{"error": "cannot link documents from different teams"}) + return + } + + link := &store.VaultLink{ + FromDocID: body.FromDocID, + ToDocID: body.ToDocID, + LinkType: body.LinkType, + Context: body.Context, + } + if err := h.store.CreateLink(r.Context(), link); err != nil { + slog.Warn("vault.create_link failed", "error", err) + writeJSON(w, http.StatusInternalServerError, map[string]string{"error": err.Error()}) + return + } + writeJSON(w, http.StatusCreated, link) +} + +// handleBatchGetLinks returns all outlinks for a batch of doc IDs in one query. +func (h *VaultHandler) handleBatchGetLinks(w http.ResponseWriter, r *http.Request) { + locale := extractLocale(r) + tenantID := store.TenantIDFromContext(r.Context()) + + var body struct { + DocIDs []string `json:"doc_ids"` + } + if !bindJSON(w, r, locale, &body) { + return + } + if len(body.DocIDs) == 0 { + writeJSON(w, http.StatusOK, []store.VaultLink{}) + return + } + if len(body.DocIDs) > 500 { + body.DocIDs = body.DocIDs[:500] + } + + links, err := h.store.GetOutLinksBatch(r.Context(), tenantID.String(), body.DocIDs) + if err != nil { + slog.Warn("vault.batch_links failed", "error", err) + writeJSON(w, http.StatusInternalServerError, map[string]string{"error": err.Error()}) + return + } + if links == nil { + links = []store.VaultLink{} + } + writeJSON(w, http.StatusOK, links) +} + +// handleDeleteLink deletes a vault link. +func (h *VaultHandler) handleDeleteLink(w http.ResponseWriter, r *http.Request) { + tenantID := store.TenantIDFromContext(r.Context()) + linkID := r.PathValue("linkID") + + if err := h.store.DeleteLink(r.Context(), tenantID.String(), linkID); err != nil { + slog.Warn("vault.delete_link failed", "error", err) + writeJSON(w, http.StatusInternalServerError, map[string]string{"error": err.Error()}) + return + } + w.WriteHeader(http.StatusNoContent) +} diff --git a/internal/http/vault_handler_upload.go b/internal/http/vault_handler_upload.go new file mode 100644 index 00000000..032a3d68 --- /dev/null +++ b/internal/http/vault_handler_upload.go @@ -0,0 +1,231 @@ +package http + +import ( + "io" + "log/slog" + "net/http" + "os" + "path/filepath" + "strings" + "time" + + "github.com/google/uuid" + "github.com/nextlevelbuilder/goclaw/internal/eventbus" + "github.com/nextlevelbuilder/goclaw/internal/store" + "github.com/nextlevelbuilder/goclaw/internal/vault" +) + +// allowedUploadExts is the whitelist of text-based file extensions accepted by vault upload. +var allowedUploadExts = map[string]bool{ + ".md": true, ".txt": true, ".json": true, ".yaml": true, ".yml": true, + ".csv": true, ".toml": true, ".xml": true, ".html": true, ".htm": true, + ".go": true, ".py": true, ".js": true, ".ts": true, ".tsx": true, ".jsx": true, + ".rs": true, ".java": true, ".rb": true, ".sh": true, ".sql": true, + ".swift": true, ".kt": true, ".c": true, ".cpp": true, ".h": true, +} + +// handleUpload accepts multipart file uploads and registers them in the vault. +// Files are written to the tenant workspace in the appropriate subfolder based on agent/team selection, +// then UpsertDocument + EventVaultDocUpserted triggers the existing enrichment pipeline. +func (h *VaultHandler) handleUpload(w http.ResponseWriter, r *http.Request) { + tenantID := store.TenantIDFromContext(r.Context()) + tenantIDStr := tenantID.String() + + // 32 MB in-memory; remainder spills to disk. + if err := r.ParseMultipartForm(32 << 20); err != nil { + writeJSON(w, http.StatusBadRequest, map[string]string{"error": "invalid multipart form"}) + return + } + defer r.MultipartForm.RemoveAll() + + agentIDStr := r.FormValue("agent_id") + teamIDStr := r.FormValue("team_id") + + // Validate team membership if provided. + if teamIDStr != "" { + if !h.validateTeamMembership(r.Context(), w, teamIDStr) { + return + } + } + + // Resolve agent UUID → agent_key for folder placement. + var agentKey string + if agentIDStr != "" { + if h.agents != nil { + agents, err := h.agents.List(r.Context(), "") + if err == nil { + for _, a := range agents { + if a.ID.String() == agentIDStr { + agentKey = a.AgentKey + break + } + } + } + } + if agentKey == "" { + writeJSON(w, http.StatusBadRequest, map[string]string{"error": "agent not found"}) + return + } + } + + // Determine target subfolder and scope. + var subDir, scope string + switch { + case agentIDStr != "": + subDir = filepath.Join("agents", agentKey) + scope = "personal" + case teamIDStr != "": + subDir = filepath.Join("teams", teamIDStr) + scope = "team" + default: + scope = "shared" + } + + wsPath := h.resolveTenantWorkspace(r.Context()) + if wsPath == "" { + writeJSON(w, http.StatusBadRequest, map[string]string{"error": "workspace not available"}) + return + } + targetDir := filepath.Join(wsPath, subDir) + if err := os.MkdirAll(targetDir, 0o755); err != nil { + slog.Warn("vault.upload: mkdir failed", "dir", targetDir, "error", err) + writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "failed to create target directory"}) + return + } + + files := r.MultipartForm.File["files"] + if len(files) == 0 { + writeJSON(w, http.StatusBadRequest, map[string]string{"error": "no files provided"}) + return + } + if len(files) > 50 { + writeJSON(w, http.StatusBadRequest, map[string]string{"error": "too many files (max 50)"}) + return + } + + type uploadResult struct { + Doc *store.VaultDocument `json:"document"` + Error string `json:"error,omitempty"` + } + + var results []uploadResult + var created int + + for _, fh := range files { + // Sanitize filename — basename only, no path traversal. + name := filepath.Base(fh.Filename) + if name == "." || name == ".." || name == "" { + results = append(results, uploadResult{Error: "invalid filename: " + fh.Filename}) + continue + } + + // Validate extension. + ext := strings.ToLower(filepath.Ext(name)) + if !allowedUploadExts[ext] { + results = append(results, uploadResult{Error: "unsupported file type: " + name}) + continue + } + + // Enforce per-file size limit (50 MB). + if fh.Size > 50<<20 { + results = append(results, uploadResult{Error: "file too large: " + name}) + continue + } + + // Open uploaded file. + src, err := fh.Open() + if err != nil { + results = append(results, uploadResult{Error: "failed to read: " + name}) + continue + } + + // Write to workspace. + dstPath := filepath.Join(targetDir, name) + dst, err := os.Create(dstPath) + if err != nil { + src.Close() + results = append(results, uploadResult{Error: "failed to write: " + name}) + continue + } + _, copyErr := io.Copy(dst, src) + dst.Close() + src.Close() + if copyErr != nil { + results = append(results, uploadResult{Error: "failed to write: " + name}) + continue + } + + // Hash the written file. + hash, err := vault.ContentHashFile(dstPath) + if err != nil { + results = append(results, uploadResult{Error: "failed to hash: " + name}) + continue + } + + // Build workspace-relative path for DB storage. + relPath := name + if subDir != "" { + relPath = subDir + "/" + name + } + + doc := &store.VaultDocument{ + TenantID: tenantIDStr, + Path: relPath, + Title: vault.InferTitle(relPath), + DocType: vault.InferDocType(relPath), + ContentHash: hash, + Scope: scope, + } + if agentIDStr != "" { + doc.AgentID = &agentIDStr + } + if teamIDStr != "" { + doc.TeamID = &teamIDStr + } + + if err := h.store.UpsertDocument(r.Context(), doc); err != nil { + slog.Warn("vault.upload: upsert failed", "path", relPath, "error", err) + results = append(results, uploadResult{Error: "failed to register: " + name}) + continue + } + + // Publish enrichment event (same pattern as rescan.go). + if h.eventBus != nil { + agentForEvent := "" + if agentIDStr != "" { + agentForEvent = agentIDStr + } + h.eventBus.Publish(eventbus.DomainEvent{ + ID: uuid.Must(uuid.NewV7()).String(), + Type: eventbus.EventVaultDocUpserted, + SourceID: doc.ID + ":" + hash, + TenantID: tenantIDStr, + AgentID: agentForEvent, + Timestamp: time.Now(), + Payload: eventbus.VaultDocUpsertedPayload{ + DocID: doc.ID, + TenantID: tenantIDStr, + AgentID: agentForEvent, + Path: relPath, + ContentHash: hash, + Workspace: wsPath, + }, + }) + } + + results = append(results, uploadResult{Doc: doc}) + created++ + } + + // Signal enrichment progress for WS subscribers. + if h.enrichProgress != nil && created > 0 { + h.enrichProgress.Start(created, tenantID) + } + + slog.Info("vault.upload", "tenant", tenantIDStr, "uploaded", created, "errors", len(files)-created) + + writeJSON(w, http.StatusOK, map[string]any{ + "documents": results, + "count": created, + }) +} diff --git a/internal/http/vault_handlers.go b/internal/http/vault_handlers.go index b5fe2a7c..ac060399 100644 --- a/internal/http/vault_handlers.go +++ b/internal/http/vault_handlers.go @@ -6,9 +6,14 @@ import ( "net/http" "strconv" "strings" + "sync" + "time" "github.com/google/uuid" + "github.com/nextlevelbuilder/goclaw/internal/config" + "github.com/nextlevelbuilder/goclaw/internal/eventbus" "github.com/nextlevelbuilder/goclaw/internal/store" + "github.com/nextlevelbuilder/goclaw/internal/vault" ) // vaultDocListResponse wraps the document list with total count for pagination. @@ -17,14 +22,33 @@ type vaultDocListResponse struct { Total int `json:"total"` } -// VaultHandler serves Knowledge Vault document and link endpoints. -type VaultHandler struct { - store store.VaultStore - teamAccess store.TeamAccessStore // nil = skip team membership validation (e.g. lite edition) +// AgentLister is the subset of AgentStore needed by VaultHandler (rescan agent_key→UUID mapping). +type AgentLister interface { + List(ctx context.Context, ownerID string) ([]store.AgentData, error) } -func NewVaultHandler(s store.VaultStore, ta store.TeamAccessStore) *VaultHandler { - return &VaultHandler{store: s, teamAccess: ta} +// TeamLister is the subset of TeamStore needed by VaultHandler (rescan team validation). +type TeamLister interface { + ListTeams(ctx context.Context) ([]store.TeamData, error) +} + +// VaultHandler serves Knowledge Vault document and link endpoints. +type VaultHandler struct { + store store.VaultStore + teamAccess store.TeamAccessStore // nil = skip team membership validation (e.g. lite edition) + agents AgentLister // nil = rescan skips agent resolution + teams TeamLister // nil = rescan skips team resolution + workspace string + eventBus eventbus.DomainEventBus + enrichProgress *vault.EnrichProgress // nil = enrichment progress SSE disabled + rescanMu sync.Map // key: tenantID → struct{}, per-tenant concurrency guard +} + +// SetEnrichProgress injects the enrichment progress tracker for SSE streaming. +func (h *VaultHandler) SetEnrichProgress(p *vault.EnrichProgress) { h.enrichProgress = p } + +func NewVaultHandler(s store.VaultStore, ta store.TeamAccessStore, workspace string, bus eventbus.DomainEventBus, agents AgentLister, teams TeamLister) *VaultHandler { + return &VaultHandler{store: s, teamAccess: ta, agents: agents, teams: teams, workspace: workspace, eventBus: bus} } // validateTeamMembership checks that the requesting user belongs to the given team. @@ -83,16 +107,28 @@ func (h *VaultHandler) applyNonOwnerTeamScope(ctx context.Context, opts *store.V } func (h *VaultHandler) RegisterRoutes(mux *http.ServeMux) { - // Cross-agent endpoint (agent_id optional query param). + // Cross-agent endpoints (agentID = "" from PathValue when no {agentID} in path). mux.HandleFunc("GET /v1/vault/documents", h.auth(h.handleListAllDocuments)) - // Per-agent endpoints. + mux.HandleFunc("POST /v1/vault/documents", h.auth(h.handleCreateDocument)) + mux.HandleFunc("GET /v1/vault/documents/{docID}", h.auth(h.handleGetDocument)) + mux.HandleFunc("PUT /v1/vault/documents/{docID}", h.auth(h.handleUpdateDocument)) + mux.HandleFunc("DELETE /v1/vault/documents/{docID}", h.auth(h.handleDeleteDocument)) + mux.HandleFunc("GET /v1/vault/documents/{docID}/links", h.auth(h.handleGetLinks)) + mux.HandleFunc("POST /v1/vault/links", h.auth(h.handleCreateLink)) + mux.HandleFunc("DELETE /v1/vault/links/{linkID}", h.auth(h.handleDeleteLink)) + mux.HandleFunc("POST /v1/vault/links/batch", h.auth(h.handleBatchGetLinks)) + mux.HandleFunc("POST /v1/vault/upload", h.auth(h.handleUpload)) + mux.HandleFunc("POST /v1/vault/rescan", h.auth(h.handleRescan)) + mux.HandleFunc("POST /v1/vault/search", h.auth(h.handleSearchAll)) + mux.HandleFunc("GET /v1/vault/enrichment/status", h.auth(h.handleEnrichmentStatus)) + // Per-agent endpoints (backward compat — same handlers, agentID from path). mux.HandleFunc("GET /v1/agents/{agentID}/vault/documents", h.auth(h.handleListDocuments)) mux.HandleFunc("GET /v1/agents/{agentID}/vault/documents/{docID}", h.auth(h.handleGetDocument)) mux.HandleFunc("POST /v1/agents/{agentID}/vault/documents", h.auth(h.handleCreateDocument)) mux.HandleFunc("PUT /v1/agents/{agentID}/vault/documents/{docID}", h.auth(h.handleUpdateDocument)) mux.HandleFunc("DELETE /v1/agents/{agentID}/vault/documents/{docID}", h.auth(h.handleDeleteDocument)) - mux.HandleFunc("POST /v1/agents/{agentID}/vault/search", h.auth(h.handleSearch)) mux.HandleFunc("GET /v1/agents/{agentID}/vault/documents/{docID}/links", h.auth(h.handleGetLinks)) + mux.HandleFunc("POST /v1/agents/{agentID}/vault/search", h.auth(h.handleSearch)) mux.HandleFunc("POST /v1/agents/{agentID}/vault/links", h.auth(h.handleCreateLink)) mux.HandleFunc("DELETE /v1/agents/{agentID}/vault/links/{linkID}", h.auth(h.handleDeleteLink)) } @@ -122,436 +158,91 @@ func (h *VaultHandler) parseListOpts(r *http.Request) store.VaultListOptions { return opts } -// handleListAllDocuments lists vault documents across all agents in tenant. -// Optional query param agent_id to filter by specific agent. -func (h *VaultHandler) handleListAllDocuments(w http.ResponseWriter, r *http.Request) { - tenantID := store.TenantIDFromContext(r.Context()) - agentID := r.URL.Query().Get("agent_id") - opts := h.parseListOpts(r) +// handleRescan walks the entire tenant workspace and registers missing/changed files in vault. +// Infers agent/team ownership from directory structure: agents/{key}/, teams/{uuid}/, or root shared. +func (h *VaultHandler) handleRescan(w http.ResponseWriter, r *http.Request) { + tenantID := store.TenantIDFromContext(r.Context()).String() - // Validate team membership if specific team requested. - if opts.TeamID != nil && *opts.TeamID != "" { - if !h.validateTeamMembership(r.Context(), w, *opts.TeamID) { - return - } + // Per-tenant concurrency guard. + if _, loaded := h.rescanMu.LoadOrStore(tenantID, struct{}{}); loaded { + writeJSON(w, http.StatusConflict, map[string]string{"error": "rescan already in progress"}) + return } - // Non-owner without team_id filter: show personal + user's teams. - if opts.TeamID == nil && !store.IsOwnerRole(r.Context()) { - h.applyNonOwnerTeamScope(r.Context(), &opts) + defer h.rescanMu.Delete(tenantID) + + wsPath := h.resolveTenantWorkspace(r.Context()) + if wsPath == "" { + writeJSON(w, http.StatusBadRequest, map[string]string{"error": "workspace not available"}) + return } - docs, err := h.store.ListDocuments(r.Context(), tenantID.String(), agentID, opts) + // Build agent_key→UUID map and team UUID set for path inference. + agentMap, teamSet := h.buildRescanMaps(r.Context()) + + ctx, cancel := context.WithTimeout(r.Context(), 5*time.Minute) + defer cancel() + + result, err := vault.RescanWorkspace(ctx, vault.RescanParams{ + TenantID: tenantID, + Workspace: wsPath, + AgentMap: agentMap, + TeamSet: teamSet, + }, h.store, h.eventBus) if err != nil { - slog.Warn("vault.list_all failed", "error", err) + slog.Warn("vault.rescan failed", "tenant", tenantID, "error", err) writeJSON(w, http.StatusInternalServerError, map[string]string{"error": err.Error()}) return } - if docs == nil { - docs = []store.VaultDocument{} + + // Signal enrichment progress so WS subscribers see running=true immediately. + if h.enrichProgress != nil && (result.New+result.Updated) > 0 { + h.enrichProgress.Start(result.New+result.Updated, store.TenantIDFromContext(r.Context())) } - total, cntErr := h.store.CountDocuments(r.Context(), tenantID.String(), agentID, opts) - if cntErr != nil { - slog.Warn("vault.count failed", "error", cntErr) - } - writeJSON(w, http.StatusOK, vaultDocListResponse{Documents: docs, Total: total}) + + writeJSON(w, http.StatusOK, result) } -// handleListDocuments lists vault documents for a specific agent. -func (h *VaultHandler) handleListDocuments(w http.ResponseWriter, r *http.Request) { - tenantID := store.TenantIDFromContext(r.Context()) - agentID := r.PathValue("agentID") - opts := h.parseListOpts(r) - - if opts.TeamID != nil && *opts.TeamID != "" { - if !h.validateTeamMembership(r.Context(), w, *opts.TeamID) { - return - } +// resolveTenantWorkspace returns the tenant-scoped workspace root. +func (h *VaultHandler) resolveTenantWorkspace(ctx context.Context) string { + if h.workspace == "" { + return "" } - if opts.TeamID == nil && !store.IsOwnerRole(r.Context()) { - h.applyNonOwnerTeamScope(r.Context(), &opts) - } - - docs, err := h.store.ListDocuments(r.Context(), tenantID.String(), agentID, opts) - if err != nil { - slog.Warn("vault.list failed", "error", err) - writeJSON(w, http.StatusInternalServerError, map[string]string{"error": err.Error()}) - return - } - if docs == nil { - docs = []store.VaultDocument{} - } - total, cntErr := h.store.CountDocuments(r.Context(), tenantID.String(), agentID, opts) - if cntErr != nil { - slog.Warn("vault.count failed", "error", cntErr) - } - writeJSON(w, http.StatusOK, vaultDocListResponse{Documents: docs, Total: total}) + tenantID := store.TenantIDFromContext(ctx) + slug := store.TenantSlugFromContext(ctx) + return config.TenantWorkspace(h.workspace, tenantID, slug) } -// handleGetDocument returns a single vault document by ID, scoped to the agent. -func (h *VaultHandler) handleGetDocument(w http.ResponseWriter, r *http.Request) { - tenantID := store.TenantIDFromContext(r.Context()) - agentID := r.PathValue("agentID") - docID := r.PathValue("docID") +// buildRescanMaps pre-loads agent_key→UUID and team UUID sets for the current tenant. +func (h *VaultHandler) buildRescanMaps(ctx context.Context) (map[string]string, map[string]bool) { + agentMap := make(map[string]string) + teamSet := make(map[string]bool) - doc, err := h.store.GetDocumentByID(r.Context(), tenantID.String(), docID) - if err != nil { - slog.Warn("vault.get failed", "error", err) - writeJSON(w, http.StatusInternalServerError, map[string]string{"error": err.Error()}) - return - } - if doc == nil || doc.AgentID != agentID { - writeJSON(w, http.StatusNotFound, map[string]string{"error": "document not found"}) - return - } - // Verify team boundary — non-owner must be team member to view team docs. - if doc.TeamID != nil && *doc.TeamID != "" && !store.IsOwnerRole(r.Context()) { - if !h.validateTeamMembership(r.Context(), w, *doc.TeamID) { - return - } - } - writeJSON(w, http.StatusOK, doc) -} - -// handleSearch runs hybrid FTS+vector search on vault documents. -func (h *VaultHandler) handleSearch(w http.ResponseWriter, r *http.Request) { - locale := extractLocale(r) - tenantID := store.TenantIDFromContext(r.Context()) - agentID := r.PathValue("agentID") - - var body struct { - Query string `json:"query"` - Scope string `json:"scope"` - DocTypes []string `json:"doc_types"` - MaxResults int `json:"max_results"` - TeamID string `json:"team_id"` - } - if !bindJSON(w, r, locale, &body) { - return - } - if body.Query == "" { - writeJSON(w, http.StatusBadRequest, map[string]string{"error": "query is required"}) - return - } - if body.MaxResults <= 0 { - body.MaxResults = 10 - } - - searchOpts := store.VaultSearchOptions{ - Query: body.Query, - AgentID: agentID, - TenantID: tenantID.String(), - Scope: body.Scope, - DocTypes: body.DocTypes, - MaxResults: body.MaxResults, - } - if body.TeamID != "" { - if !h.validateTeamMembership(r.Context(), w, body.TeamID) { - return - } - searchOpts.TeamID = &body.TeamID - } else if !store.IsOwnerRole(r.Context()) { - if ids := h.userAccessibleTeamIDs(r.Context()); len(ids) > 0 { - searchOpts.TeamIDs = ids - } else { - empty := "" - searchOpts.TeamID = &empty - } - } - - results, err := h.store.Search(r.Context(), searchOpts) - if err != nil { - slog.Warn("vault.search failed", "error", err) - writeJSON(w, http.StatusInternalServerError, map[string]string{"error": err.Error()}) - return - } - if results == nil { - results = []store.VaultSearchResult{} - } - writeJSON(w, http.StatusOK, results) -} - -// handleGetLinks returns outgoing links and backlinks for a vault document. -func (h *VaultHandler) handleGetLinks(w http.ResponseWriter, r *http.Request) { - tenantID := store.TenantIDFromContext(r.Context()) - _ = r.PathValue("agentID") // agent scoping done at document level - docID := r.PathValue("docID") - - outLinks, err := h.store.GetOutLinks(r.Context(), tenantID.String(), docID) - if err != nil { - slog.Warn("vault.outlinks failed", "error", err) - writeJSON(w, http.StatusInternalServerError, map[string]string{"error": err.Error()}) - return - } - backlinks, err := h.store.GetBacklinks(r.Context(), tenantID.String(), docID) - if err != nil { - slog.Warn("vault.backlinks failed", "error", err) - writeJSON(w, http.StatusInternalServerError, map[string]string{"error": err.Error()}) - return - } - if outLinks == nil { - outLinks = []store.VaultLink{} - } - if backlinks == nil { - backlinks = []store.VaultBacklink{} - } - - // Filter backlinks by team boundary — derive team context from the target document - // itself (not a query param) so clients don't need to supply it correctly. - isOwner := store.IsOwnerRole(r.Context()) - if !isOwner { - targetDoc, _ := h.store.GetDocumentByID(r.Context(), tenantID.String(), docID) - var currentTeamID string - if targetDoc != nil && targetDoc.TeamID != nil { - currentTeamID = *targetDoc.TeamID - } - filtered := make([]store.VaultBacklink, 0, len(backlinks)) - for _, bl := range backlinks { - if currentTeamID != "" { - if bl.TeamID != nil && *bl.TeamID != currentTeamID { - continue - } - } else { - if bl.TeamID != nil && *bl.TeamID != "" { - continue - } + if h.agents != nil { + agents, err := h.agents.List(ctx, "") + if err == nil { + for _, a := range agents { + agentMap[a.AgentKey] = a.ID.String() } - filtered = append(filtered, bl) } - backlinks = filtered } - - writeJSON(w, http.StatusOK, map[string]any{ - "outlinks": outLinks, - "backlinks": backlinks, - }) + if h.teams != nil { + teams, err := h.teams.ListTeams(ctx) + if err == nil { + for _, t := range teams { + teamSet[t.ID.String()] = true + } + } + } + return agentMap, teamSet } -// handleCreateDocument creates a new vault document. -func (h *VaultHandler) handleCreateDocument(w http.ResponseWriter, r *http.Request) { - locale := extractLocale(r) - tenantID := store.TenantIDFromContext(r.Context()) - agentID := r.PathValue("agentID") - - var body struct { - Path string `json:"path"` - Title string `json:"title"` - DocType string `json:"doc_type"` - Scope string `json:"scope"` - TeamID string `json:"team_id"` - Metadata map[string]any `json:"metadata"` - } - if !bindJSON(w, r, locale, &body) { +// handleEnrichmentStatus returns the current enrichment pipeline progress as JSON. +func (h *VaultHandler) handleEnrichmentStatus(w http.ResponseWriter, r *http.Request) { + if h.enrichProgress == nil { + writeJSON(w, http.StatusOK, vault.EnrichEvent{}) return } - if body.Path == "" || body.Title == "" { - writeJSON(w, http.StatusBadRequest, map[string]string{"error": "path and title are required"}) - return - } - if body.DocType == "" { - body.DocType = "note" - } - if body.Scope == "" { - body.Scope = "personal" - } - if !validDocType(body.DocType) { - writeJSON(w, http.StatusBadRequest, map[string]string{"error": "invalid doc_type"}) - return - } - if !validScope(body.Scope) { - writeJSON(w, http.StatusBadRequest, map[string]string{"error": "invalid scope"}) - return - } - - doc := &store.VaultDocument{ - TenantID: tenantID.String(), - AgentID: agentID, - Path: body.Path, - Title: body.Title, - DocType: body.DocType, - Scope: body.Scope, - Metadata: body.Metadata, - } - if body.TeamID != "" { - if !h.validateTeamMembership(r.Context(), w, body.TeamID) { - return - } - doc.TeamID = &body.TeamID - if body.Scope == "personal" { - doc.Scope = "team" - } - } - if err := h.store.UpsertDocument(r.Context(), doc); err != nil { - slog.Warn("vault.create failed", "error", err) - writeJSON(w, http.StatusInternalServerError, map[string]string{"error": err.Error()}) - return - } - // Re-fetch by ID (set via RETURNING) — unambiguous even when same path exists across teams. - created, _ := h.store.GetDocumentByID(r.Context(), tenantID.String(), doc.ID) - if created != nil { - writeJSON(w, http.StatusCreated, created) - } else { - writeJSON(w, http.StatusCreated, doc) - } -} - -// handleUpdateDocument updates an existing vault document. -func (h *VaultHandler) handleUpdateDocument(w http.ResponseWriter, r *http.Request) { - locale := extractLocale(r) - tenantID := store.TenantIDFromContext(r.Context()) - agentID := r.PathValue("agentID") - docID := r.PathValue("docID") - - existing, err := h.store.GetDocumentByID(r.Context(), tenantID.String(), docID) - if err != nil || existing == nil || existing.AgentID != agentID { - writeJSON(w, http.StatusNotFound, map[string]string{"error": "document not found"}) - return - } - - var body struct { - Title *string `json:"title"` - DocType *string `json:"doc_type"` - Scope *string `json:"scope"` - TeamID *string `json:"team_id"` // nil=no change, ""=clear, "uuid"=set - Metadata map[string]any `json:"metadata"` - } - if !bindJSON(w, r, locale, &body) { - return - } - - if body.Title != nil { - existing.Title = *body.Title - } - if body.DocType != nil { - existing.DocType = *body.DocType - } - if body.Scope != nil { - existing.Scope = *body.Scope - } - if body.TeamID != nil { - // Only owner/admin can change team assignment. - if !store.IsOwnerRole(r.Context()) { - writeJSON(w, http.StatusForbidden, map[string]string{"error": "only owner can change document team assignment"}) - return - } - if *body.TeamID == "" { - existing.TeamID = nil - existing.Scope = "personal" - } else { - existing.TeamID = body.TeamID - existing.Scope = "team" - } - } - if body.Metadata != nil { - existing.Metadata = body.Metadata - } - - if err := h.store.UpsertDocument(r.Context(), existing); err != nil { - slog.Warn("vault.update failed", "error", err) - writeJSON(w, http.StatusInternalServerError, map[string]string{"error": err.Error()}) - return - } - updated, _ := h.store.GetDocumentByID(r.Context(), tenantID.String(), docID) - if updated != nil { - writeJSON(w, http.StatusOK, updated) - } else { - writeJSON(w, http.StatusOK, existing) - } -} - -// handleDeleteDocument deletes a vault document and its links. -func (h *VaultHandler) handleDeleteDocument(w http.ResponseWriter, r *http.Request) { - tenantID := store.TenantIDFromContext(r.Context()) - agentID := r.PathValue("agentID") - docID := r.PathValue("docID") - - existing, err := h.store.GetDocumentByID(r.Context(), tenantID.String(), docID) - if err != nil || existing == nil || existing.AgentID != agentID { - writeJSON(w, http.StatusNotFound, map[string]string{"error": "document not found"}) - return - } - - // Verify team boundary before deletion. - if existing.TeamID != nil && *existing.TeamID != "" && !store.IsOwnerRole(r.Context()) { - if !h.validateTeamMembership(r.Context(), w, *existing.TeamID) { - return - } - } - - // DeleteDocument without RunContext applies no team_id filter (broad match on tenant+agent+path). - // This is safe because we pre-validated team membership above and use server-derived existing.Path. - if err := h.store.DeleteDocument(r.Context(), tenantID.String(), agentID, existing.Path); err != nil { - slog.Warn("vault.delete failed", "error", err) - writeJSON(w, http.StatusInternalServerError, map[string]string{"error": err.Error()}) - return - } - w.WriteHeader(http.StatusNoContent) -} - -// handleCreateLink creates a link between two vault documents. -func (h *VaultHandler) handleCreateLink(w http.ResponseWriter, r *http.Request) { - locale := extractLocale(r) - tenantID := store.TenantIDFromContext(r.Context()) - - var body struct { - FromDocID string `json:"from_doc_id"` - ToDocID string `json:"to_doc_id"` - LinkType string `json:"link_type"` - Context string `json:"context"` - } - if !bindJSON(w, r, locale, &body) { - return - } - if body.FromDocID == "" || body.ToDocID == "" { - writeJSON(w, http.StatusBadRequest, map[string]string{"error": "from_doc_id and to_doc_id are required"}) - return - } - if body.LinkType == "" { - body.LinkType = "reference" - } - - // Verify both docs exist, same tenant, and at least source belongs to this agent. - agentID := r.PathValue("agentID") - from, _ := h.store.GetDocumentByID(r.Context(), tenantID.String(), body.FromDocID) - to, _ := h.store.GetDocumentByID(r.Context(), tenantID.String(), body.ToDocID) - if from == nil || to == nil { - writeJSON(w, http.StatusNotFound, map[string]string{"error": "one or both documents not found"}) - return - } - if from.AgentID != agentID { - writeJSON(w, http.StatusForbidden, map[string]string{"error": "source document does not belong to this agent"}) - return - } - // Block cross-team linking (both team docs must be in same team). - if from.TeamID != nil && to.TeamID != nil && *from.TeamID != *to.TeamID { - writeJSON(w, http.StatusBadRequest, map[string]string{"error": "cannot link documents from different teams"}) - return - } - - link := &store.VaultLink{ - FromDocID: body.FromDocID, - ToDocID: body.ToDocID, - LinkType: body.LinkType, - Context: body.Context, - } - if err := h.store.CreateLink(r.Context(), link); err != nil { - slog.Warn("vault.create_link failed", "error", err) - writeJSON(w, http.StatusInternalServerError, map[string]string{"error": err.Error()}) - return - } - writeJSON(w, http.StatusCreated, link) -} - -// handleDeleteLink deletes a vault link. -func (h *VaultHandler) handleDeleteLink(w http.ResponseWriter, r *http.Request) { - tenantID := store.TenantIDFromContext(r.Context()) - linkID := r.PathValue("linkID") - - if err := h.store.DeleteLink(r.Context(), tenantID.String(), linkID); err != nil { - slog.Warn("vault.delete_link failed", "error", err) - writeJSON(w, http.StatusInternalServerError, map[string]string{"error": err.Error()}) - return - } - w.WriteHeader(http.StatusNoContent) + writeJSON(w, http.StatusOK, h.enrichProgress.Status()) } var allowedDocTypes = map[string]bool{"context": true, "memory": true, "note": true, "skill": true, "episodic": true, "media": true} diff --git a/internal/mcp/bridge_tool.go b/internal/mcp/bridge_tool.go index 5d3215cc..69026a86 100644 --- a/internal/mcp/bridge_tool.go +++ b/internal/mcp/bridge_tool.go @@ -15,6 +15,8 @@ import ( // BridgeTool adapts an MCP tool into the tools.Tool interface. // It delegates Execute calls to the MCP server via the client. +// The client pointer is loaded atomically from clientPtr to support +// safe reconnection without data races. type BridgeTool struct { serverName string toolName string // original MCP tool name @@ -22,7 +24,7 @@ type BridgeTool struct { description string inputSchema map[string]any // JSON Schema for parameters requiredSet map[string]bool - client *mcpclient.Client + clientPtr *atomic.Pointer[mcpclient.Client] // shared with serverState for atomic swap on reconnect timeoutSec int connected *atomic.Bool } @@ -30,7 +32,9 @@ type BridgeTool struct { // NewBridgeTool creates a BridgeTool from an MCP Tool definition. // The tool name is always prefixed with "mcp_" to distinguish MCP tools from native tools. // If prefix is empty, it is auto-derived from the server name. -func NewBridgeTool(serverName string, mcpTool mcpgo.Tool, client *mcpclient.Client, prefix string, timeoutSec int, connected *atomic.Bool) *BridgeTool { +// clientPtr is a shared atomic pointer from serverState — reconnection swaps it +// atomically, and all BridgeTools see the new client without explicit notification. +func NewBridgeTool(serverName string, mcpTool mcpgo.Tool, clientPtr *atomic.Pointer[mcpclient.Client], prefix string, timeoutSec int, connected *atomic.Bool) *BridgeTool { name := mcpTool.Name effectivePrefix := ensureMCPPrefix(prefix, serverName) registered := effectivePrefix + "__" + name @@ -53,7 +57,7 @@ func NewBridgeTool(serverName string, mcpTool mcpgo.Tool, client *mcpclient.Clie description: mcpTool.Description, inputSchema: schema, requiredSet: reqSet, - client: client, + clientPtr: clientPtr, timeoutSec: timeoutSec, connected: connected, } @@ -99,6 +103,11 @@ func (t *BridgeTool) Execute(ctx context.Context, args map[string]any) *tools.Re return tools.ErrorResult(fmt.Sprintf("MCP server %q is disconnected", t.serverName)) } + client := t.clientPtr.Load() // atomic load — safe during concurrent reconnect + if client == nil { + return tools.ErrorResult(fmt.Sprintf("MCP server %q has no active client", t.serverName)) + } + callCtx, cancel := context.WithTimeout(ctx, time.Duration(t.timeoutSec)*time.Second) defer cancel() @@ -111,7 +120,7 @@ func (t *BridgeTool) Execute(ctx context.Context, args map[string]any) *tools.Re req.Params.Name = t.toolName req.Params.Arguments = cleanedArgs - result, err := t.client.CallTool(callCtx, req) + result, err := client.CallTool(callCtx, req) if err != nil { if errors.Is(callCtx.Err(), context.DeadlineExceeded) { return tools.ErrorResult(fmt.Sprintf("MCP tool %q timeout after %ds", t.registeredName, t.timeoutSec)) diff --git a/internal/mcp/manager.go b/internal/mcp/manager.go index c5ebab06..903e693a 100644 --- a/internal/mcp/manager.go +++ b/internal/mcp/manager.go @@ -25,6 +25,7 @@ const ( initialBackoff = 2 * time.Second maxBackoff = 60 * time.Second maxReconnectAttempts = 10 + reconnectCooldown = 5 * time.Minute // wait after exhausting reconnect attempts before retrying // mcpToolInlineMaxCount is the threshold above which MCP tools switch // to search mode (deferred loading via mcp_tool_search) instead of @@ -41,15 +42,35 @@ type ServerStatus struct { Error string `json:"error,omitempty"` } +// connParams stores connection parameters needed to re-establish a dead connection. +// Populated during initial connectAndDiscover and used by tryReconnect. +type connParams struct { + command string + args []string + env map[string]string + url string + headers map[string]string +} + // serverState tracks a single MCP server connection. +// +// Dual-pointer design for the MCP client: +// - client: direct pointer used by healthLoop (single goroutine, no contention). +// - clientPtr: atomic pointer shared with all BridgeTools via NewBridgeTool. +// BridgeTools call clientPtr.Load() in Execute for race-safe access. +// +// On reconnect, fullReconnect() updates BOTH: ss.client for healthLoop and +// ss.clientPtr.Store() for BridgeTools. The old client is closed AFTER the swap. type serverState struct { name string transport string - client *mcpclient.Client + client *mcpclient.Client // direct ref for health checks (single-goroutine access) + clientPtr atomic.Pointer[mcpclient.Client] // shared atomic ref for BridgeTools (multi-goroutine safe) connected atomic.Bool toolNames []string // registered tool names in the registry timeoutSec int cancel context.CancelFunc + conn connParams // connection params for reconnect mu sync.Mutex reconnAttempts int @@ -495,8 +516,9 @@ func (m *Manager) Stop() { if ss.cancel != nil { ss.cancel() } - if ss.client != nil { - if err := ss.client.Close(); err != nil { + // Use atomic pointer — health loop may swap client via fullReconnect concurrently. + if client := ss.clientPtr.Load(); client != nil { + if err := client.Close(); err != nil { slog.Debug("mcp.server.close_error", "server", name, "error", err) } } diff --git a/internal/mcp/manager_connect.go b/internal/mcp/manager_connect.go index 9bb6972a..03a60257 100644 --- a/internal/mcp/manager_connect.go +++ b/internal/mcp/manager_connect.go @@ -86,7 +86,15 @@ func connectAndDiscover(ctx context.Context, name, transportType, command string transport: transportType, client: client, timeoutSec: timeoutSec, + conn: connParams{ + command: command, + args: args, + env: env, + url: url, + headers: headers, + }, } + ss.clientPtr.Store(client) ss.connected.Store(true) return ss, toolsResult.Tools, nil @@ -133,7 +141,7 @@ func (m *Manager) connectServer(ctx context.Context, name, transportType, comman func (m *Manager) registerBridgeTools(ss *serverState, mcpTools []mcpgo.Tool, serverName, toolPrefix string, timeoutSec int) []string { var registeredNames []string for _, mcpTool := range mcpTools { - bt := NewBridgeTool(serverName, mcpTool, ss.client, toolPrefix, timeoutSec, &ss.connected) + bt := NewBridgeTool(serverName, mcpTool, &ss.clientPtr, toolPrefix, timeoutSec, &ss.connected) if _, exists := m.registry.Get(bt.Name()); exists { slog.Warn("mcp.tool.name_collision", @@ -199,7 +207,7 @@ func (m *Manager) connectViaPool(ctx context.Context, tenantID uuid.UUID, name, func (m *Manager) registerPoolBridgeTools(entry *poolEntry, serverName, toolPrefix string, timeoutSec int) []string { var registeredNames []string for _, mcpTool := range entry.tools { - bt := NewBridgeTool(serverName, mcpTool, entry.state.client, toolPrefix, timeoutSec, &entry.state.connected) + bt := NewBridgeTool(serverName, mcpTool, &entry.state.clientPtr, toolPrefix, timeoutSec, &entry.state.connected) if _, exists := m.registry.Get(bt.Name()); exists { slog.Warn("mcp.tool.name_collision", @@ -302,24 +310,36 @@ func (m *Manager) healthLoop(ctx context.Context, ss *serverState) { // tryReconnect attempts to reconnect with exponential backoff. func (m *Manager) tryReconnect(ctx context.Context, ss *serverState) { + reconnectWithBackoff(ctx, ss, "mcp.server") +} + +// reconnectWithBackoff implements the two-phase reconnect strategy shared by +// Manager.healthLoop and poolHealthLoop. Handles cooldown after exhausting +// max attempts, exponential backoff, fast-path ping (transient blips), and +// slow-path full reconnect (dead server-side session). +// logPrefix distinguishes log entries (e.g. "mcp.server" vs "mcp.pool"). +func reconnectWithBackoff(ctx context.Context, ss *serverState, logPrefix string) { ss.mu.Lock() if ss.reconnAttempts >= maxReconnectAttempts { - ss.lastErr = fmt.Sprintf("max reconnect attempts (%d) reached", maxReconnectAttempts) + ss.lastErr = fmt.Sprintf("max reconnect attempts (%d) reached, entering cooldown", maxReconnectAttempts) ss.mu.Unlock() - slog.Error("mcp.server.reconnect_exhausted", "server", ss.name) - return + slog.Warn(logPrefix+".reconnect_cooldown", "server", ss.name, "cooldown", reconnectCooldown) + select { + case <-ctx.Done(): + return + case <-time.After(reconnectCooldown): + } + ss.mu.Lock() + ss.reconnAttempts = 0 + ss.mu.Unlock() + return // will retry on next health tick } ss.reconnAttempts++ attempt := ss.reconnAttempts ss.mu.Unlock() backoff := min(initialBackoff*time.Duration(1<<(attempt-1)), maxBackoff) - - slog.Info("mcp.server.reconnecting", - "server", ss.name, - "attempt", attempt, - "backoff", backoff, - ) + slog.Info(logPrefix+".reconnecting", "server", ss.name, "attempt", attempt, "backoff", backoff) select { case <-ctx.Done(): @@ -327,13 +347,74 @@ func (m *Manager) tryReconnect(ctx context.Context, ss *serverState) { case <-time.After(backoff): } - // Try to ping again — transport may have auto-reconnected + // Fast path: ping existing client — works for transient network blips + // where the server-side session is still alive. if err := ss.client.Ping(ctx); err == nil { ss.connected.Store(true) ss.mu.Lock() ss.reconnAttempts = 0 + ss.healthFailures = 0 ss.lastErr = "" ss.mu.Unlock() - slog.Info("mcp.server.reconnected", "server", ss.name) + slog.Info(logPrefix+".reconnected", "server", ss.name) + return + } + + // Slow path: server-side session is dead (container restart, OOM, etc.). + if fullReconnect(ctx, ss) { + slog.Info(logPrefix+".reconnected", "server", ss.name, "method", "full_reconnect") } } + +// fullReconnect creates a fresh MCP client, atomically swaps it into serverState, +// and closes the old one. Returns true on success. Used by reconnectWithBackoff +// as the slow path when pinging the old client fails. +// +// The new client is created and validated FIRST, then swapped via clientPtr.Store() +// so BridgeTools see the new client immediately. The old client is closed AFTER +// the swap to avoid a window where ss.client points to a closed client. +// +// NOTE: Does not re-discover tools (ListTools). If the MCP server restarts with +// a different tool set, changes won't be reflected until the Manager reconnects. +func fullReconnect(ctx context.Context, ss *serverState) bool { + slog.Info("mcp.full_reconnect", "server", ss.name, "transport", ss.transport) + + newClient, err := createClient(ss.transport, ss.conn.command, ss.conn.args, ss.conn.env, ss.conn.url, ss.conn.headers) + if err != nil { + slog.Warn("mcp.reconnect_create_failed", "server", ss.name, "error", err) + return false + } + + if ss.transport != "stdio" { + if err := newClient.Start(ctx); err != nil { + _ = newClient.Close() + slog.Warn("mcp.reconnect_start_failed", "server", ss.name, "error", err) + return false + } + } + + initReq := mcpgo.InitializeRequest{} + initReq.Params.ProtocolVersion = mcpgo.LATEST_PROTOCOL_VERSION + initReq.Params.ClientInfo = mcpgo.Implementation{Name: "goclaw", Version: "1.0.0"} + + if _, err := newClient.Initialize(ctx, initReq); err != nil { + _ = newClient.Close() + slog.Warn("mcp.reconnect_init_failed", "server", ss.name, "error", err) + return false + } + + // Swap atomically: store new client, then close old. + // BridgeTools use clientPtr.Load() so they see the new client immediately. + oldClient := ss.client + ss.client = newClient + ss.clientPtr.Store(newClient) + ss.connected.Store(true) + ss.mu.Lock() + ss.reconnAttempts = 0 + ss.healthFailures = 0 + ss.lastErr = "" + ss.mu.Unlock() + + _ = oldClient.Close() + return true +} diff --git a/internal/mcp/pool.go b/internal/mcp/pool.go index 2d19a332..b767f35f 100644 --- a/internal/mcp/pool.go +++ b/internal/mcp/pool.go @@ -131,8 +131,8 @@ func (p *Pool) Acquire(ctx context.Context, tenantID uuid.UUID, name, transportT if old.state.cancel != nil { old.state.cancel() } - if old.state.client != nil { - _ = old.state.client.Close() + if client := old.state.clientPtr.Load(); client != nil { + _ = client.Close() } delete(p.servers, key) // Return slot to semaphore @@ -216,8 +216,8 @@ func (p *Pool) AcquireUser(ctx context.Context, tenantID uuid.UUID, name, userID if old.state.cancel != nil { old.state.cancel() } - if old.state.client != nil { - _ = old.state.client.Close() + if client := old.state.clientPtr.Load(); client != nil { + _ = client.Close() } delete(p.userServers, key) // Return slot to per-server semaphore @@ -390,8 +390,8 @@ func (p *Pool) Stop() { if entry.state.cancel != nil { entry.state.cancel() } - if entry.state.client != nil { - _ = entry.state.client.Close() + if client := entry.state.clientPtr.Load(); client != nil { + _ = client.Close() } slog.Debug("mcp.pool.stopped", "key", key) } @@ -401,8 +401,8 @@ func (p *Pool) Stop() { if entry.state.cancel != nil { entry.state.cancel() } - if entry.state.client != nil { - _ = entry.state.client.Close() + if client := entry.state.clientPtr.Load(); client != nil { + _ = client.Close() } slog.Debug("mcp.pool.user.stopped", "key", key) } @@ -423,8 +423,8 @@ func (p *Pool) Evict(tenantID uuid.UUID, serverName string) { if entry.state.cancel != nil { entry.state.cancel() } - if entry.state.client != nil { - _ = entry.state.client.Close() + if client := entry.state.clientPtr.Load(); client != nil { + _ = client.Close() } delete(p.servers, key) select { @@ -481,8 +481,8 @@ func (p *Pool) evictIdle() { if entry.state.cancel != nil { entry.state.cancel() } - if entry.state.client != nil { - _ = entry.state.client.Close() + if client := entry.state.clientPtr.Load(); client != nil { + _ = client.Close() } delete(p.servers, key) select { @@ -499,8 +499,8 @@ func (p *Pool) evictIdle() { if entry.state.cancel != nil { entry.state.cancel() } - if entry.state.client != nil { - _ = entry.state.client.Close() + if client := entry.state.clientPtr.Load(); client != nil { + _ = client.Close() } delete(p.userServers, key) // Return slot to per-server semaphore @@ -588,8 +588,9 @@ func (p *Pool) evictOldestIdleLocked() bool { return true } -// Client returns the MCP client for this pool entry. -func (e *poolEntry) Client() *mcpclient.Client { return e.state.client } +// ClientPtr returns the atomic client pointer for this pool entry. +// Used by BridgeTools to atomically load the current client during reconnect. +func (e *poolEntry) ClientPtr() *atomic.Pointer[mcpclient.Client] { return &e.state.clientPtr } // Connected returns a pointer to the connected flag for this pool entry. func (e *poolEntry) Connected() *atomic.Bool { return &e.state.connected } @@ -598,6 +599,8 @@ func (e *poolEntry) Connected() *atomic.Bool { return &e.state.connected } func (e *poolEntry) MCPTools() []mcpgo.Tool { return e.tools } // poolHealthLoop is a standalone health loop for pool-managed connections. +// After consecutive ping failures, it attempts a full reconnect by creating +// a fresh client, mirroring the Manager.tryReconnect slow path. func poolHealthLoop(ctx context.Context, ss *serverState) { ticker := newHealthTicker() defer ticker.Stop() @@ -625,6 +628,7 @@ func poolHealthLoop(ctx context.Context, ss *serverState) { if failures >= healthFailThreshold { ss.connected.Store(false) + poolTryReconnect(ctx, ss) } } else { ss.connected.Store(true) @@ -637,3 +641,9 @@ func poolHealthLoop(ctx context.Context, ss *serverState) { } } } + +// poolTryReconnect attempts reconnect for a pool-managed connection. +// Delegates to the shared reconnectWithBackoff with pool-specific log prefix. +func poolTryReconnect(ctx context.Context, ss *serverState) { + reconnectWithBackoff(ctx, ss, "mcp.pool") +} diff --git a/internal/memory/auto_injector.go b/internal/memory/auto_injector.go index 618ae69f..724ef4e2 100644 --- a/internal/memory/auto_injector.go +++ b/internal/memory/auto_injector.go @@ -20,6 +20,18 @@ type InjectParams struct { UserID string TenantID string UserMessage string + + // RecentContext carries a short snippet of recent conversation (typically + // the last 1-2 user turns concatenated) used to enrich the search query. + // Context-aware recall: without this, vector search on "what's my favorite?" + // misses memories about the topic under discussion. With it, the query + // embedding captures conversational intent and returns materially better + // matches for follow-up questions. + // + // Empty = legacy behaviour (search on UserMessage only). + // Target length: ≤ ~400 chars. Longer context dilutes the embedding. + RecentContext string + MaxEntries int // default 5 MaxTokens int // default 200 Threshold float64 // relevance threshold (default 0.3) diff --git a/internal/memory/auto_injector_impl.go b/internal/memory/auto_injector_impl.go index 1b9693b9..29871655 100644 --- a/internal/memory/auto_injector_impl.go +++ b/internal/memory/auto_injector_impl.go @@ -41,8 +41,15 @@ func (a *pgAutoInjector) Inject(ctx context.Context, params InjectParams) (*Inje threshold = 0.3 } + // Phase 9: context-aware recall. When the caller supplied RecentContext, + // build a richer search query that captures conversational intent. Without + // this, vector search on "what's my favorite?" misses memories about the + // topic under discussion. With it, the query embedding captures the + // follow-up semantics and returns materially better matches. + searchQuery := buildRecallQuery(params.UserMessage, params.RecentContext) + // Search with FTS bias (faster than vector for auto-inject) - results, err := a.episodicStore.Search(ctx, params.UserMessage, params.AgentID, params.UserID, + results, err := a.episodicStore.Search(ctx, searchQuery, params.AgentID, params.UserID, store.EpisodicSearchOptions{ MaxResults: maxEntries * 2, // fetch more, filter by threshold MinScore: threshold, diff --git a/internal/memory/recall_query.go b/internal/memory/recall_query.go new file mode 100644 index 00000000..6631ed6c --- /dev/null +++ b/internal/memory/recall_query.go @@ -0,0 +1,62 @@ +package memory + +import "strings" + +// maxRecallContextRunes bounds the recent-context snippet used to enrich the +// recall query. Longer snippets dilute the embedding signal and slow down the +// vector search; shorter snippets lose the conversational frame. +// +// 400 runes ≈ 100 tokens in Latin scripts, fewer in CJK (1-2 short user turns +// either way). Tuning knob; change only if recall quality metrics show a +// clear trend in either direction. +// +// Unit is runes, not bytes, because GoClaw supports vi/zh locales: a +// byte-wise tail-clip would slice a multi-byte rune in half and emit invalid +// UTF-8 to the embedding model. +const maxRecallContextRunes = 400 + +// buildRecallQuery concatenates the user's latest message with a short recent +// context snippet to produce a context-aware search query. Used by auto-inject +// to improve recall on follow-up questions where the current message alone is +// ambiguous (pronouns, implicit references, one-word replies). +// +// The recent context is truncated to maxRecallContextRunes (rune-safe for +// CJK/vi/zh input) and prepended so embedding models give the latest message +// the most weight (position bias). Empty context or empty message return the +// unmodified input — zero-risk fallback for legacy callers that don't supply +// RecentContext yet. +func buildRecallQuery(userMessage, recentContext string) string { + userMessage = strings.TrimSpace(userMessage) + if recentContext == "" || userMessage == "" { + return userMessage + } + + ctx := strings.TrimSpace(recentContext) + ctx = tailClipRunes(ctx, maxRecallContextRunes) + + // Prepend context with a lightweight separator. "Context:" and "Query:" + // tags help instruction-tuned embedding models distinguish frame from + // focus; for non-instruction models they act as neutral separators. + var sb strings.Builder + sb.Grow(len(userMessage) + len(ctx) + 32) + sb.WriteString("Context: ") + sb.WriteString(ctx) + sb.WriteString("\nQuery: ") + sb.WriteString(userMessage) + return sb.String() +} + +// tailClipRunes returns the last maxRunes runes of s, rune-safe for multi-byte +// scripts (Vietnamese, Chinese, Japanese, emoji). If s has fewer runes than +// maxRunes it's returned unchanged. Byte-wise slicing would split multi-byte +// runes and emit invalid UTF-8. +func tailClipRunes(s string, maxRunes int) string { + if maxRunes <= 0 { + return "" + } + runes := []rune(s) + if len(runes) <= maxRunes { + return s + } + return string(runes[len(runes)-maxRunes:]) +} diff --git a/internal/memory/recall_query_test.go b/internal/memory/recall_query_test.go new file mode 100644 index 00000000..438e865c --- /dev/null +++ b/internal/memory/recall_query_test.go @@ -0,0 +1,120 @@ +package memory + +import ( + "strings" + "testing" +) + +func TestBuildRecallQuery_NoContextReturnsMessageUnchanged(t *testing.T) { + got := buildRecallQuery("what's my favorite?", "") + if got != "what's my favorite?" { + t.Errorf("empty context should return message unchanged, got %q", got) + } +} + +func TestBuildRecallQuery_EmptyMessageReturnsEmpty(t *testing.T) { + got := buildRecallQuery("", "some recent context") + if got != "" { + t.Errorf("empty message should return empty, got %q", got) + } +} + +func TestBuildRecallQuery_CombinesContextAndMessage(t *testing.T) { + ctx := "We were talking about coffee shops in downtown." + msg := "what's my favorite?" + got := buildRecallQuery(msg, ctx) + + if !strings.Contains(got, msg) { + t.Errorf("result must contain user message, got %q", got) + } + if !strings.Contains(got, ctx) { + t.Errorf("result must contain recent context, got %q", got) + } + // Query label must come AFTER context so position-biased embeddings give + // the message the strongest signal. + ctxIdx := strings.Index(got, "Context:") + queryIdx := strings.Index(got, "Query:") + if ctxIdx == -1 || queryIdx == -1 || ctxIdx >= queryIdx { + t.Errorf("Context must precede Query label, got %q", got) + } +} + +func TestBuildRecallQuery_TruncatesOversizedContext(t *testing.T) { + // 1000-char context, maxRecallContextChars = 400 + ctx := strings.Repeat("a", 1000) + msg := "test" + got := buildRecallQuery(msg, ctx) + + // Stripped context section should be at most maxRecallContextRunes long. + // Full output has prefix overhead ("Context: " + "\nQuery: " + msg) ~ 20 chars. + // Total should be < 500 chars (bounded by truncation + prefix + msg). + if len(got) > maxRecallContextRunes+64 { + t.Errorf("oversized context not truncated: got %d chars, want ≤ %d", + len(got), maxRecallContextRunes+64) + } +} + +func TestBuildRecallQuery_TailClipsLongContext(t *testing.T) { + // Tail-clip keeps the most recent portion (last N chars) since it's + // closer to the current turn in conversation time. + ctx := strings.Repeat("OLD ", 200) + strings.Repeat("NEW ", 20) // 820 chars, "NEW" at the end + got := buildRecallQuery("follow-up", ctx) + if !strings.Contains(got, "NEW") { + t.Errorf("tail-clip should preserve recent ('NEW') portion, got %q", got) + } +} + +func TestBuildRecallQuery_TrimsWhitespace(t *testing.T) { + got := buildRecallQuery(" query ", " ctx ") + if !strings.Contains(got, "Context: ctx") { + t.Errorf("context should be trimmed, got %q", got) + } + if !strings.Contains(got, "Query: query") { + t.Errorf("message should be trimmed, got %q", got) + } +} + +// TestBuildRecallQuery_UnicodeSafeTailClip verifies rune-safe truncation for +// Vietnamese / Chinese input. Byte-wise slicing would produce invalid UTF-8 at +// multi-byte rune boundaries — embedding models on that garbage return lower +// quality matches, defeating Phase 9's whole point. +func TestBuildRecallQuery_UnicodeSafeTailClip(t *testing.T) { + // 500 Vietnamese tiếng Việt runes (each char = 2-3 bytes) + viChar := "ế" // 3 bytes in UTF-8 + ctx := strings.Repeat(viChar, 500) + msg := "câu hỏi" + got := buildRecallQuery(msg, ctx) + // The result must contain valid UTF-8 — no half-runes at the clip boundary. + // If the clip cut a rune, the resulting string would contain 0xEF 0xBF 0xBD + // replacement chars or fail strings.ContainsRune checks. + if !strings.Contains(got, msg) { + t.Errorf("message missing from result, got %q", got) + } + // Must still contain the Vietnamese marker char + if !strings.Contains(got, viChar) { + t.Errorf("vietnamese context missing, got %q", got) + } + // Every rune must be valid (conversion round-trip) + for _, r := range got { + if r == '\uFFFD' { + t.Errorf("result contains replacement rune (invalid UTF-8 clip boundary), got %q", got) + break + } + } +} + +func TestTailClipRunes(t *testing.T) { + if got := tailClipRunes("hello", 10); got != "hello" { + t.Errorf("short input should be unchanged, got %q", got) + } + if got := tailClipRunes("abcdef", 3); got != "def" { + t.Errorf("expected 'def', got %q", got) + } + // Chinese: 10 Han chars, clip to 3 + if got := tailClipRunes("一二三四五六七八九十", 3); got != "八九十" { + t.Errorf("expected '八九十', got %q", got) + } + if got := tailClipRunes("anything", 0); got != "" { + t.Errorf("zero maxRunes should return empty, got %q", got) + } +} diff --git a/internal/pipeline/context_stage.go b/internal/pipeline/context_stage.go index 51441f82..1a97faeb 100644 --- a/internal/pipeline/context_stage.go +++ b/internal/pipeline/context_stage.go @@ -3,6 +3,7 @@ package pipeline import ( "context" "fmt" + "strings" "github.com/nextlevelbuilder/goclaw/internal/bootstrap" "github.com/nextlevelbuilder/goclaw/internal/providers" @@ -34,6 +35,21 @@ func (s *ContextStage) Execute(ctx context.Context, state *RunState) error { state.Ctx = ctx } + // 0.5. Resolve the effective context window for this run's provider/model. + // Done once here so PruneStage reads a stable value on every iteration and + // the budget can't drift if the model somehow changes mid-run. A zero + // result from the resolver (unknown model, no registry) leaves the field + // zero — PruneStage then falls back to Config.ContextWindow. + if s.deps.ResolveContextWindow != nil && state.Model != "" { + providerID := "" + if state.Provider != nil { + providerID = state.Provider.Name() + } + if cw := s.deps.ResolveContextWindow(providerID, state.Model); cw > 0 { + state.Context.EffectiveContextWindow = cw + } + } + // 1. Resolve workspace if s.deps.ResolveWorkspace != nil { ws, err := s.deps.ResolveWorkspace(ctx, state.Input) @@ -96,8 +112,11 @@ func (s *ContextStage) Execute(ctx context.Context, state *RunState) error { // 8. Auto-inject L0 memory context into system prompt. // V3RetrievalEnabled check removed — auto-inject runs whenever AutoInject is available. + // Phase 9: pass recent conversation context so vector search can resolve + // pronouns and implicit references in follow-up questions. if s.deps.AutoInject != nil && state.Input.Message != "" { - section, err := s.deps.AutoInject(ctx, state.Input.Message, state.Input.UserID) + recentCtx := buildRecentContext(state.Messages.History()) + section, err := s.deps.AutoInject(ctx, state.Input.Message, state.Input.UserID, recentCtx) if err == nil && section != "" { state.Context.MemorySection = section sys := state.Messages.System() @@ -109,6 +128,50 @@ func (s *ContextStage) Execute(ctx context.Context, state *RunState) error { return nil } +// buildRecentContext concatenates the trailing user turns from the history +// buffer into a short snippet suitable for enriching a recall query. Walks +// backward from the end so we keep the most recent turns regardless of +// earlier messages being pruned. Returns "" when there's no usable context. +// +// Budget: up to 2 user turns, max ~300 runes total. Rune (not byte) cap keeps +// vi/zh locales safe — a byte-wise clip would slice multi-byte characters +// and emit invalid UTF-8 to the embedding model. Tuning knob is intentional +// here rather than config-driven — Phase 9 adds it only if operational data +// shows variance across agent types. +func buildRecentContext(history []providers.Message) string { + const maxTurns = 2 + const maxRunes = 300 + if len(history) == 0 { + return "" + } + turns := make([]string, 0, maxTurns) + for i := len(history) - 1; i >= 0 && len(turns) < maxTurns; i-- { + m := history[i] + if m.Role != "user" || m.Content == "" { + continue + } + turns = append([]string{m.Content}, turns...) // prepend to preserve order + } + if len(turns) == 0 { + return "" + } + var sb strings.Builder + for i, t := range turns { + if i > 0 { + sb.WriteString(" | ") + } + sb.WriteString(t) + } + joined := sb.String() + // Rune-safe tail clip: keep the most recent portion of the conversation + // (closest in time to the current turn) rather than the oldest. + runes := []rune(joined) + if len(runes) > maxRunes { + joined = string(runes[len(runes)-maxRunes:]) + } + return joined +} + // toAnySlice converts []bootstrap.ContextFile to []any for ContextState.ContextFiles. // Phase 8 will remove this when ContextState uses typed field. func toAnySlice(files []bootstrap.ContextFile) []any { diff --git a/internal/pipeline/deps.go b/internal/pipeline/deps.go index 255da902..64f21939 100644 --- a/internal/pipeline/deps.go +++ b/internal/pipeline/deps.go @@ -18,12 +18,20 @@ type PipelineDeps struct { EventBus eventbus.DomainEventBus Config PipelineConfig + // ResolveContextWindow returns the effective context window (in tokens) for + // a given provider/model pair. Nil = always use Config.ContextWindow. + // Invoked ONCE per run by ContextStage and stored in RunState.Context.EffectiveContextWindow. + ResolveContextWindow func(provider, model string) int + // Callbacks from agent.Loop — Phase 8 adapter wires these. EmitEvent func(event any) // Auto-inject memory context (ContextStage, L0 tier). - // Callback captures agent/tenant context via closure. - AutoInject func(ctx context.Context, userMessage, userID string) (string, error) + // Callback captures agent/tenant context via closure. recentContext carries + // a short snippet of recent conversation (last 1-2 user turns) so the + // downstream recall query can resolve pronouns and implicit references. + // Empty recentContext = legacy single-message search semantics. + AutoInject func(ctx context.Context, userMessage, userID, recentContext string) (string, error) // InjectContext sets up agent/tenant/user/workspace/tool context values. // Wraps injectContext() for v3 pipeline. Called once at ContextStage start. @@ -92,6 +100,14 @@ type PipelineConfig struct { MaxTokens int Compaction *config.CompactionConfig + // ReserveTokens is a safety buffer subtracted from the history budget so + // PruneStage compacts slightly before the hard limit. Prevents edge cases + // where a provider returns more than MaxTokens output or where the token + // counter's estimate drifts upward during streaming. + // Zero (default) preserves legacy behavior: budget = contextWindow - overhead - MaxTokens. + // Recommended: 5-10% of contextWindow for reasoning-heavy models. + ReserveTokens int + // V3 memory/retrieval flags removed — always true at runtime. // Memory flush runs if callback != nil; auto-inject runs if AutoInject != nil. } diff --git a/internal/pipeline/prune_stage.go b/internal/pipeline/prune_stage.go index f525477c..e8de2b16 100644 --- a/internal/pipeline/prune_stage.go +++ b/internal/pipeline/prune_stage.go @@ -31,8 +31,19 @@ func (s *PruneStage) Result() StageResult { return s.result } func (s *PruneStage) Execute(ctx context.Context, state *RunState) error { s.result = Continue - // Compute budget: context window minus overhead (system prompt + context files) minus output reserve - budget := s.deps.Config.ContextWindow - state.Context.OverheadTokens - s.deps.Config.MaxTokens + // Compute budget using the effective context window for this run's model. + // ContextStage resolves EffectiveContextWindow once per run via ModelRegistry; + // if zero (unknown model, registry not wired) fall back to the pipeline-wide + // Config.ContextWindow for backward compatibility. + // + // ReserveTokens (optional, default 0) carves out a safety buffer so compaction + // fires slightly before the hard limit — protects against provider over-delivery + // and token-counter drift on streaming responses. + contextWindow := state.Context.EffectiveContextWindow + if contextWindow == 0 { + contextWindow = s.deps.Config.ContextWindow + } + budget := contextWindow - state.Context.OverheadTokens - s.deps.Config.MaxTokens - s.deps.Config.ReserveTokens if budget <= 0 { return nil // no history budget, nothing to prune } diff --git a/internal/pipeline/stages_test.go b/internal/pipeline/stages_test.go index fa7a75c9..86fce166 100644 --- a/internal/pipeline/stages_test.go +++ b/internal/pipeline/stages_test.go @@ -5,6 +5,7 @@ import ( "errors" "os" "path/filepath" + "strings" "testing" "github.com/nextlevelbuilder/goclaw/internal/bootstrap" @@ -500,6 +501,193 @@ func TestPruneStage_ZeroBudget_NoOp(t *testing.T) { } } +// TestPruneStage_EffectiveContextWindow_OverridesConfig verifies that when +// ContextStage has resolved a per-model context window (e.g. gpt-4o=128k +// vs Config.ContextWindow=10k), PruneStage uses the resolved value. +// +// Without this, swapping models at session time would leave the pipeline +// pruning against the wrong budget. Phase 4 regression gate. +func TestPruneStage_EffectiveContextWindow_OverridesConfig(t *testing.T) { + t.Parallel() + pruneCallCount := 0 + // Config says 10k window, but model-specific resolution says 128k. + // History is 60k tokens — would be over budget at 10k (fires prune), + // under budget at 128k (no prune should happen). + deps := &PipelineDeps{ + Config: PipelineConfig{ + ContextWindow: 10_000, // stale/default — should be ignored + MaxTokens: 1_000, + }, + TokenCounter: &mockTokenCounter{countPerMessage: 600}, + PruneMessages: func(msgs []providers.Message, _ int) []providers.Message { + pruneCallCount++ + return msgs + }, + } + stage := NewPruneStage(deps, nil) + state := defaultState() + state.Context.EffectiveContextWindow = 128_000 // resolved by ContextStage + + history := make([]providers.Message, 100) // 100 * 600 = 60k < budget 127k + for i := range history { + history[i] = providers.Message{Role: "user", Content: "msg"} + } + state.Messages.SetHistory(history) + + if err := stage.Execute(context.Background(), state); err != nil { + t.Fatalf("Execute() error: %v", err) + } + if pruneCallCount != 0 { + t.Errorf("PruneMessages should not fire with 128k window, called %d times", pruneCallCount) + } +} + +// --- buildRecentContext tests (Phase 9) --- + +func TestBuildRecentContext_EmptyHistory(t *testing.T) { + t.Parallel() + if got := buildRecentContext(nil); got != "" { + t.Errorf("empty history should return empty, got %q", got) + } +} + +func TestBuildRecentContext_SkipsNonUserMessages(t *testing.T) { + t.Parallel() + hist := []providers.Message{ + {Role: "user", Content: "hello"}, + {Role: "assistant", Content: "hi back"}, + {Role: "tool", Content: "tool output"}, + } + got := buildRecentContext(hist) + if got != "hello" { + t.Errorf("should only include user messages, got %q", got) + } +} + +func TestBuildRecentContext_CapsAtTwoTurns(t *testing.T) { + t.Parallel() + hist := []providers.Message{ + {Role: "user", Content: "first"}, + {Role: "assistant", Content: "reply"}, + {Role: "user", Content: "second"}, + {Role: "assistant", Content: "reply"}, + {Role: "user", Content: "third"}, + } + got := buildRecentContext(hist) + // Should contain last two user turns ("second" and "third"), not "first" + if !strings.Contains(got, "second") { + t.Errorf("missing second turn, got %q", got) + } + if !strings.Contains(got, "third") { + t.Errorf("missing third turn, got %q", got) + } + if strings.Contains(got, "first") { + t.Errorf("should exclude old turns, got %q", got) + } +} + +func TestBuildRecentContext_PreservesTurnOrder(t *testing.T) { + t.Parallel() + hist := []providers.Message{ + {Role: "user", Content: "earlier"}, + {Role: "user", Content: "later"}, + } + got := buildRecentContext(hist) + // "earlier" should come before "later" in the output + earlierIdx := strings.Index(got, "earlier") + laterIdx := strings.Index(got, "later") + if earlierIdx == -1 || laterIdx == -1 || earlierIdx >= laterIdx { + t.Errorf("order broken: got %q (earlier=%d, later=%d)", got, earlierIdx, laterIdx) + } +} + +func TestBuildRecentContext_TruncatesLongMessages(t *testing.T) { + t.Parallel() + long := strings.Repeat("x", 500) + hist := []providers.Message{ + {Role: "user", Content: long}, + } + got := buildRecentContext(hist) + if len(got) > 300 { + t.Errorf("result should be capped at 300 chars, got %d", len(got)) + } +} + +// --- Prune Stage tests continued --- + +// TestPruneStage_ReserveTokens_BuffersBudget verifies that ReserveTokens is +// subtracted from the usable budget so compaction fires earlier than the hard +// limit. Phase 5 regression gate for provider over-delivery protection. +func TestPruneStage_ReserveTokens_BuffersBudget(t *testing.T) { + t.Parallel() + pruneCallCount := 0 + // Without reserve: budget = 10000 - 0 - 1000 = 9000, softThreshold 6300 + // With reserve=2000: budget = 10000 - 0 - 1000 - 2000 = 7000, softThreshold 4900 + // 50 msgs * 100 = 5000 tokens — crosses the 4900 soft threshold only when reserve is set. + deps := &PipelineDeps{ + Config: PipelineConfig{ + ContextWindow: 10_000, + MaxTokens: 1_000, + ReserveTokens: 2_000, + }, + TokenCounter: &mockTokenCounter{countPerMessage: 100}, + PruneMessages: func(msgs []providers.Message, _ int) []providers.Message { + pruneCallCount++ + return msgs[:1] + }, + } + stage := NewPruneStage(deps, nil) + state := defaultState() + history := make([]providers.Message, 50) + for i := range history { + history[i] = providers.Message{Role: "user", Content: "msg"} + } + state.Messages.SetHistory(history) + + if err := stage.Execute(context.Background(), state); err != nil { + t.Fatalf("Execute() error: %v", err) + } + if pruneCallCount != 1 { + t.Errorf("ReserveTokens should have triggered early prune, called %d times", pruneCallCount) + } +} + +// TestPruneStage_EffectiveContextWindow_ZeroFallsBackToConfig verifies +// backward compatibility: when ContextStage hasn't resolved a model-specific +// window (unknown model, nil resolver, no registry), PruneStage falls back +// to Config.ContextWindow — matching pre-Phase-4 behavior. +func TestPruneStage_EffectiveContextWindow_ZeroFallsBackToConfig(t *testing.T) { + t.Parallel() + pruneCallCount := 0 + deps := &PipelineDeps{ + Config: PipelineConfig{ + ContextWindow: 10_000, // used because EffectiveContextWindow=0 + MaxTokens: 1_000, + }, + TokenCounter: &mockTokenCounter{countPerMessage: 600}, + PruneMessages: func(msgs []providers.Message, _ int) []providers.Message { + pruneCallCount++ + return msgs[:1] + }, + } + stage := NewPruneStage(deps, nil) + state := defaultState() + // state.Context.EffectiveContextWindow left as zero + + history := make([]providers.Message, 20) // 20 * 600 = 12k > 10k → should prune + for i := range history { + history[i] = providers.Message{Role: "user", Content: "msg"} + } + state.Messages.SetHistory(history) + + if err := stage.Execute(context.Background(), state); err != nil { + t.Fatalf("Execute() error: %v", err) + } + if pruneCallCount == 0 { + t.Error("PruneMessages should fire with 10k fallback window") + } +} + // --- ToolStage tests --- func TestToolStage_NoToolCalls_NoOp(t *testing.T) { diff --git a/internal/pipeline/substates.go b/internal/pipeline/substates.go index ef15c578..6bc3f0e5 100644 --- a/internal/pipeline/substates.go +++ b/internal/pipeline/substates.go @@ -15,6 +15,16 @@ type ContextState struct { Summary string // session summary for context continuity HadBootstrap bool OverheadTokens int // system prompt + context files (accurate via TokenCounter) + + // EffectiveContextWindow is the context window size (in tokens) resolved + // per-run from the provider/model pair via ModelRegistry. Resolved ONCE in + // ContextStage and read by PruneStage on every iteration. Zero means "no + // model-specific data available" and PruneStage falls back to + // PipelineConfig.ContextWindow. + // + // Resolved once per run (not per iteration) to avoid budget skew — if the + // model somehow changes mid-run a mismatch causes silent truncation loops. + EffectiveContextWindow int } // ThinkState: owned by ThinkStage. diff --git a/internal/providers/anthropic.go b/internal/providers/anthropic.go index ad46661b..bafc267c 100644 --- a/internal/providers/anthropic.go +++ b/internal/providers/anthropic.go @@ -138,20 +138,29 @@ func (p *AnthropicProvider) Chat(ctx context.Context, req ChatRequest) (*ChatRes body := p.buildRequestBody(model, req, false) body = ApplyMiddlewares(body, p.middlewares, p.middlewareConfig(model, req)) - return RetryDo(ctx, p.retryConfig, func() (*ChatResponse, error) { + resp, err := RetryDo(ctx, p.retryConfig, func() (*ChatResponse, error) { respBody, err := p.doRequest(ctx, body) if err != nil { return nil, err } defer respBody.Close() - var resp anthropicResponse - if err := json.NewDecoder(respBody).Decode(&resp); err != nil { + var parsed anthropicResponse + if err := json.NewDecoder(respBody).Decode(&parsed); err != nil { return nil, fmt.Errorf("anthropic: decode response: %w", err) } - return p.parseResponse(&resp), nil + return p.parseResponse(&parsed), nil }) + // Drop user-visible reasoning after parsing for models flagged as leakers. + // Usage.ThinkingTokens and RawAssistantContent remain intact so billing + // and Anthropic tool-use thinking passback continue to work. + if resp != nil { + if strip, _ := req.Options[OptStripThinking].(bool); strip { + resp.Thinking = "" + } + } + return resp, err } func (p *AnthropicProvider) doRequest(ctx context.Context, body any) (io.ReadCloser, error) { diff --git a/internal/providers/anthropic_stream.go b/internal/providers/anthropic_stream.go index 7d6861de..128fa803 100644 --- a/internal/providers/anthropic_stream.go +++ b/internal/providers/anthropic_stream.go @@ -10,6 +10,11 @@ import ( func (p *AnthropicProvider) ChatStream(ctx context.Context, req ChatRequest, onChunk func(StreamChunk)) (*ChatResponse, error) { model := resolveAnthropicModel(req.Model, p.defaultModel, p.registry) + // stripThinking: when true, drop reasoning tokens from user-visible output. + // Billing counters (thinkingChars → Usage.ThinkingTokens) and tool-passback + // RawAssistantContent remain untouched so billing and Anthropic's thinking + // block replay continue to work. + stripThinking, _ := req.Options[OptStripThinking].(bool) body := p.buildRequestBody(model, req, true) body = ApplyMiddlewares(body, p.middlewares, p.middlewareConfig(model, req)) @@ -80,10 +85,14 @@ func (p *AnthropicProvider) ChatStream(ctx context.Context, req ChatRequest, onC onChunk(StreamChunk{Content: ev.Delta.Text}) } case "thinking_delta": - result.Thinking += ev.Delta.Thinking + // Always count raw thinking bytes for billing estimation + // below, even when stripping user-visible output. thinkingChars += len(ev.Delta.Thinking) - if onChunk != nil { - onChunk(StreamChunk{Thinking: ev.Delta.Thinking}) + if !stripThinking { + result.Thinking += ev.Delta.Thinking + if onChunk != nil { + onChunk(StreamChunk{Thinking: ev.Delta.Thinking}) + } } case "input_json_delta": if len(result.ToolCalls) > 0 { diff --git a/internal/providers/codex.go b/internal/providers/codex.go index 66fa17d7..5d988144 100644 --- a/internal/providers/codex.go +++ b/internal/providers/codex.go @@ -118,6 +118,10 @@ func (p *CodexProvider) middlewareConfig(req ChatRequest) MiddlewareConfig { } func (p *CodexProvider) ChatStream(ctx context.Context, req ChatRequest, onChunk func(StreamChunk)) (*ChatResponse, error) { + // stripThinking: drop reasoning summaries from ChatResponse.Thinking and + // onChunk callbacks. Usage.ThinkingTokens is still populated from the + // final response.usage payload (Phase 1 billing accuracy). + stripThinking, _ := req.Options[OptStripThinking].(bool) body := p.buildRequestBody(req, true) body = ApplyMiddlewares(body, p.middlewares, p.middlewareConfig(req)) @@ -142,7 +146,7 @@ func (p *CodexProvider) ChatStream(ctx context.Context, req ChatRequest, onChunk continue } - p.processSSEEvent(&event, result, toolCalls, streamState, onChunk) + p.processSSEEvent(&event, result, toolCalls, streamState, onChunk, stripThinking) } if err := sse.Err(); err != nil { @@ -181,7 +185,9 @@ func (p *CodexProvider) ChatStream(ctx context.Context, req ChatRequest, onChunk } // processSSEEvent handles a single SSE event during streaming. -func (p *CodexProvider) processSSEEvent(event *codexSSEEvent, result *ChatResponse, toolCalls map[string]*codexToolCallAcc, streamState *codexMessageStreamState, onChunk func(StreamChunk)) { +// stripThinking drops reasoning summaries from user-visible output while +// leaving billing counters (Usage.ThinkingTokens) untouched. +func (p *CodexProvider) processSSEEvent(event *codexSSEEvent, result *ChatResponse, toolCalls map[string]*codexToolCallAcc, streamState *codexMessageStreamState, onChunk func(StreamChunk), stripThinking bool) { switch event.Type { case "response.output_item.added": if event.Item != nil { @@ -228,11 +234,13 @@ func (p *CodexProvider) processSSEEvent(event *codexSSEEvent, result *ChatRespon } toolCalls[event.Item.ID] = acc case "reasoning": - for _, s := range event.Item.Summary { - if s.Text != "" { - result.Thinking += s.Text - if onChunk != nil { - onChunk(StreamChunk{Thinking: s.Text}) + if !stripThinking { + for _, s := range event.Item.Summary { + if s.Text != "" { + result.Thinking += s.Text + if onChunk != nil { + onChunk(StreamChunk{Thinking: s.Text}) + } } } } diff --git a/internal/providers/openai_chat.go b/internal/providers/openai_chat.go index daf8ee3d..ae9505f0 100644 --- a/internal/providers/openai_chat.go +++ b/internal/providers/openai_chat.go @@ -24,7 +24,16 @@ func (p *OpenAIProvider) Chat(ctx context.Context, req ChatRequest) (*ChatRespon if err != nil { if clamped := clampMaxTokensFromError(err, body); clamped { slog.Info("max_tokens clamped, retrying", "model", model, "limit", clampedLimit(body)) - return RetryDo(ctx, p.retryConfig, chatFn) + resp, err = RetryDo(ctx, p.retryConfig, chatFn) + } + } + + // Drop user-visible reasoning for models flagged as leakers (e.g. Kimi, + // DeepSeek-Reasoner). Usage.ThinkingTokens is preserved so billing stays + // correct (Phase 1 depends on this). + if resp != nil { + if strip, _ := req.Options[OptStripThinking].(bool); strip { + resp.Thinking = "" } } @@ -52,6 +61,9 @@ func (p *OpenAIProvider) chatRequestFn(ctx context.Context, body map[string]any) func (p *OpenAIProvider) ChatStream(ctx context.Context, req ChatRequest, onChunk func(StreamChunk)) (*ChatResponse, error) { model := p.resolveModel(req.Model) + // stripThinking suppresses user-visible reasoning while leaving + // Usage.ThinkingTokens untouched (the usage chunk below still records it). + stripThinking, _ := req.Options[OptStripThinking].(bool) body := p.buildRequestBody(model, req, true) body = ApplyMiddlewares(body, p.middlewares, p.middlewareConfig(model, req)) @@ -112,7 +124,7 @@ func (p *OpenAIProvider) ChatStream(ctx context.Context, req ChatRequest, onChun if reasoning == "" { reasoning = delta.Reasoning } - if reasoning != "" { + if reasoning != "" && !stripThinking { result.Thinking += reasoning if onChunk != nil { onChunk(StreamChunk{Thinking: reasoning}) diff --git a/internal/providers/reasoning_resolution.go b/internal/providers/reasoning_resolution.go index 0725208e..d02e05e1 100644 --- a/internal/providers/reasoning_resolution.go +++ b/internal/providers/reasoning_resolution.go @@ -17,9 +17,34 @@ type ReasoningDecision struct { KnownModel bool `json:"known_model,omitempty"` SupportedLevels []string `json:"supported_levels,omitempty"` UsedProviderDefault bool `json:"used_provider_default,omitempty"` + // StripThinking signals that ChatResponse.Thinking should be dropped by + // stream handlers (model leaks reasoning even when effort="off"). + // Usage.ThinkingTokens is preserved for billing accuracy. + StripThinking bool `json:"strip_thinking,omitempty"` } -func ResolveReasoningDecision(provider Provider, model, requestedEffort, fallback, source string) ReasoningDecision { +// modelLeaksReasoning returns true for models known to emit reasoning tokens +// into the user-visible content even when reasoning effort is "off". Stream +// handlers drop these from ChatResponse.Thinking to avoid leaking raw CoT to +// end users. Billing tokens (Usage.ThinkingTokens) are NOT affected. +func modelLeaksReasoning(model string) bool { + m := strings.ToLower(strings.TrimSpace(model)) + if m == "" { + return false + } + // Known leakers: Moonshot Kimi (any variant), DeepSeek-Reasoner family. + // Extend this list only after confirming the model does not honor effort="off". + return strings.Contains(m, "kimi") || strings.Contains(m, "deepseek-reasoner") +} + +func ResolveReasoningDecision(provider Provider, model, requestedEffort, fallback, source string) (out ReasoningDecision) { + // Post-process: when resolved effort is "off" on a known leaky model, + // mark the decision for downstream stream handlers to strip thinking output. + defer func() { + if out.EffectiveEffort == "off" && modelLeaksReasoning(model) { + out.StripThinking = true + } + }() decision := ReasoningDecision{ Source: normalizeReasoningSource(source), RequestedEffort: NormalizeReasoningEffort(requestedEffort), diff --git a/internal/providers/reasoning_strip_stream_test.go b/internal/providers/reasoning_strip_stream_test.go new file mode 100644 index 00000000..51fe3dc4 --- /dev/null +++ b/internal/providers/reasoning_strip_stream_test.go @@ -0,0 +1,102 @@ +package providers + +import ( + "context" + "testing" +) + +// TestAnthropicStream_StripThinkingDropsReasoningButKeepsUsage verifies that +// when OptStripThinking is set, thinking_delta events do NOT accumulate into +// ChatResponse.Thinking but Usage.ThinkingTokens is still estimated from the +// raw byte count (billing accuracy — Phase 1 depends on this). +func TestAnthropicStream_StripThinkingDropsReasoningButKeepsUsage(t *testing.T) { + events := []string{ + "event: message_start\n", + `data: {"message":{"usage":{"input_tokens":10}}}` + "\n\n", + + "event: content_block_start\n", + `data: {"index":0,"content_block":{"type":"thinking","thinking":""}}` + "\n\n", + + "event: content_block_delta\n", + `data: {"index":0,"delta":{"type":"thinking_delta","thinking":"raw chain of thought output that must be stripped"}}` + "\n\n", + + "event: content_block_stop\n", + "data: {}\n\n", + + "event: content_block_start\n", + `data: {"index":1,"content_block":{"type":"text","text":""}}` + "\n\n", + + "event: content_block_delta\n", + `data: {"index":1,"delta":{"type":"text_delta","text":"answer"}}` + "\n\n", + + "event: content_block_stop\n", + "data: {}\n\n", + + "event: message_delta\n", + `data: {"delta":{"stop_reason":"end_turn"},"usage":{"output_tokens":20}}` + "\n\n", + + "event: message_stop\n", + "data: {}\n\n", + } + server := newAnthropicSSEServer(t, events) + p := newTestAnthropicProvider(server.URL) + + var chunks []StreamChunk + req := ChatRequest{ + Model: "kimi-k2", + Messages: []Message{{Role: "user", Content: "hello"}}, + Options: map[string]any{ + OptStripThinking: true, + }, + } + result, err := p.ChatStream(context.Background(), req, func(chunk StreamChunk) { + chunks = append(chunks, chunk) + }) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if result.Thinking != "" { + t.Errorf("Thinking = %q, want empty (stripped)", result.Thinking) + } + if result.Content != "answer" { + t.Errorf("Content = %q, want %q (content must still flow through)", result.Content, "answer") + } + // Usage.ThinkingTokens should still be counted from raw bytes: + // len("raw chain of thought output that must be stripped") = 49 → 49/4 = 12 tokens. + if result.Usage == nil || result.Usage.ThinkingTokens == 0 { + t.Errorf("Usage.ThinkingTokens = 0, want >0 (billing must still count stripped thinking)") + } + // No thinking chunks should have been emitted to the caller. + for _, c := range chunks { + if c.Thinking != "" { + t.Errorf("unexpected thinking chunk emitted while stripping: %q", c.Thinking) + } + } +} + +// TestAnthropicStream_StripThinkingFalseKeepsReasoning is the backward-compat +// smoke test: when the flag is absent, existing behaviour is unchanged. +func TestAnthropicStream_StripThinkingFalseKeepsReasoning(t *testing.T) { + events := []string{ + "event: content_block_delta\n", + `data: {"index":0,"delta":{"type":"thinking_delta","thinking":"visible thought"}}` + "\n\n", + "event: content_block_stop\n", + "data: {}\n\n", + "event: message_stop\n", + "data: {}\n\n", + } + server := newAnthropicSSEServer(t, events) + p := newTestAnthropicProvider(server.URL) + + req := ChatRequest{ + Model: "claude-sonnet-4-5-20250929", + Messages: []Message{{Role: "user", Content: "hello"}}, + } + result, err := p.ChatStream(context.Background(), req, nil) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if result.Thinking != "visible thought" { + t.Errorf("Thinking = %q, want %q (no strip flag set)", result.Thinking, "visible thought") + } +} diff --git a/internal/providers/reasoning_strip_test.go b/internal/providers/reasoning_strip_test.go new file mode 100644 index 00000000..3d94291b --- /dev/null +++ b/internal/providers/reasoning_strip_test.go @@ -0,0 +1,67 @@ +package providers + +import ( + "testing" +) + +// TestResolveReasoningDecisionStripsForLeakyModels verifies that known-leaky +// models (Kimi, DeepSeek-Reasoner) flag StripThinking when effort resolves to +// "off", so stream handlers can drop raw CoT from user-visible output. +func TestResolveReasoningDecisionStripsForLeakyModels(t *testing.T) { + cases := []struct { + name string + model string + wantStrip bool + wantEffort string + }{ + {"kimi k2 default off", "kimi-k2", true, "off"}, + {"kimi thinking preview", "moonshot/kimi-k2-thinking", true, "off"}, + {"deepseek reasoner", "deepseek-reasoner", true, "off"}, + {"deepseek chat not flagged", "deepseek-chat", false, "off"}, + {"gpt-5.4 not flagged", "gpt-5.4", false, "off"}, + {"empty model", "", false, "off"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + decision := ResolveReasoningDecision( + testThinkingProvider{thinking: true}, + tc.model, "off", "downgrade", "reasoning", + ) + if decision.EffectiveEffort != tc.wantEffort { + t.Fatalf("EffectiveEffort = %q, want %q", decision.EffectiveEffort, tc.wantEffort) + } + if decision.StripThinking != tc.wantStrip { + t.Fatalf("StripThinking = %v, want %v", decision.StripThinking, tc.wantStrip) + } + }) + } +} + +// TestResolveReasoningDecisionNoStripWhenEffortActive ensures that when a user +// explicitly asks for reasoning on a leaky model, StripThinking stays false +// (user wants to see the reasoning output). +func TestResolveReasoningDecisionNoStripWhenEffortActive(t *testing.T) { + decision := ResolveReasoningDecision( + testThinkingProvider{thinking: true}, + "kimi-k2", "high", "downgrade", "reasoning", + ) + if decision.StripThinking { + t.Fatalf("StripThinking = true, want false (effort is active)") + } +} + +// TestModelLeaksReasoning spot-checks the model allowlist directly. +func TestModelLeaksReasoning(t *testing.T) { + leaky := []string{"kimi-k2", "KIMI-K2-Thinking", "deepseek-reasoner"} + for _, m := range leaky { + if !modelLeaksReasoning(m) { + t.Errorf("modelLeaksReasoning(%q) = false, want true", m) + } + } + safe := []string{"", "gpt-5.4", "claude-4-opus", "deepseek-chat", "o3-mini"} + for _, m := range safe { + if modelLeaksReasoning(m) { + t.Errorf("modelLeaksReasoning(%q) = true, want false", m) + } + } +} diff --git a/internal/providers/types.go b/internal/providers/types.go index 9d7d4a95..abf0fdf4 100644 --- a/internal/providers/types.go +++ b/internal/providers/types.go @@ -14,6 +14,10 @@ const ( OptReasoningEffort = "reasoning_effort" OptEnableThinking = "enable_thinking" OptThinkingBudget = "thinking_budget" + // OptStripThinking (bool) tells stream handlers to drop reasoning tokens + // from ChatResponse.Thinking and onChunk callbacks. Usage.ThinkingTokens + // and RawAssistantContent are preserved (billing + tool passback safety). + OptStripThinking = "strip_thinking" // Middleware-related options (Phase 2 will use these) OptServiceTier = "service_tier" diff --git a/internal/store/contact_collector.go b/internal/store/contact_collector.go index 02e760d3..b825df5d 100644 --- a/internal/store/contact_collector.go +++ b/internal/store/contact_collector.go @@ -26,7 +26,16 @@ func NewContactCollector(s ContactStore, c cache.Cache[bool]) *ContactCollector // contactType: "user" (individual sender), "group" (group chat entity), or "topic" (forum topic). // Pass empty threadID/threadType for base contacts (DM, group root). func (c *ContactCollector) EnsureContact(ctx context.Context, channelType, channelInstance, senderID, userID, displayName, username, peerKind, contactType, threadID, threadType string) { - key := channelType + ":" + senderID + ":" + threadID + // Cache key must include every dimension the underlying DB unique constraint + // uses, otherwise dedup skips legitimate upserts: + // - tenantID: fixes cross-tenant leak (same sender in tenant A vs B) + // - channelInstance: fixes collision when two bots in the same tenant share + // overlapping sender ID spaces (e.g. two Telegram bot tokens with users + // who happen to have the same Telegram user_id) + // - threadID: different threads/topics track separate contacts + // Zero UUID (Desktop / single-tenant) keeps legacy dedup semantics intact. + tid := TenantIDFromContext(ctx) + key := tid.String() + ":" + channelType + ":" + channelInstance + ":" + senderID + ":" + threadID if _, ok := c.seen.Get(ctx, key); ok { return } @@ -34,7 +43,13 @@ func (c *ContactCollector) EnsureContact(ctx context.Context, channelType, chann contactType = "user" } if err := c.store.UpsertContact(ctx, channelType, channelInstance, senderID, userID, displayName, username, peerKind, contactType, threadID, threadType); err != nil { - slog.Warn("contact_collector.upsert_failed", "error", err, "channel", channelType, "sender", senderID) + slog.Warn("contact_collector.upsert_failed", + "error", err, + "tenant_id", tid, + "channel", channelType, + "instance", channelInstance, + "sender", senderID, + ) return } c.seen.Set(ctx, key, true, contactSeenTTL) diff --git a/internal/store/contact_collector_test.go b/internal/store/contact_collector_test.go new file mode 100644 index 00000000..561dedbf --- /dev/null +++ b/internal/store/contact_collector_test.go @@ -0,0 +1,169 @@ +package store + +import ( + "context" + "sync" + "testing" + + "github.com/google/uuid" + "github.com/nextlevelbuilder/goclaw/internal/cache" +) + +// mockContactStore records every UpsertContact call for assertion. +// Only implements the methods used by ContactCollector (store.ContactStore). +type mockContactStore struct { + mu sync.Mutex + upserts []mockUpsertCall +} + +type mockUpsertCall struct { + tenantID uuid.UUID + channelType string + channelInstance string + senderID string + userID string + displayName string + username string + peerKind string + contactType string + threadID string + threadType string +} + +func (m *mockContactStore) UpsertContact(ctx context.Context, channelType, channelInstance, senderID, userID, displayName, username, peerKind, contactType, threadID, threadType string) error { + m.mu.Lock() + defer m.mu.Unlock() + m.upserts = append(m.upserts, mockUpsertCall{ + tenantID: TenantIDFromContext(ctx), + channelType: channelType, + channelInstance: channelInstance, + senderID: senderID, + userID: userID, + displayName: displayName, + username: username, + peerKind: peerKind, + contactType: contactType, + threadID: threadID, + threadType: threadType, + }) + return nil +} + +func (m *mockContactStore) ResolveTenantUserID(_ context.Context, _, _ string) (string, error) { + return "", nil +} + +// Stub methods to satisfy ContactStore interface (not used in these tests). +func (m *mockContactStore) ListContacts(_ context.Context, _ ContactListOpts) ([]ChannelContact, error) { + return nil, nil +} +func (m *mockContactStore) CountContacts(_ context.Context, _ ContactListOpts) (int, error) { + return 0, nil +} +func (m *mockContactStore) GetContactsBySenderIDs(_ context.Context, _ []string) (map[string]ChannelContact, error) { + return nil, nil +} +func (m *mockContactStore) GetContactByID(_ context.Context, _ uuid.UUID) (*ChannelContact, error) { + return nil, nil +} +func (m *mockContactStore) GetSenderIDsByContactIDs(_ context.Context, _ []uuid.UUID) ([]string, error) { + return nil, nil +} +func (m *mockContactStore) MergeContacts(_ context.Context, _ []uuid.UUID, _ uuid.UUID) error { + return nil +} +func (m *mockContactStore) UnmergeContacts(_ context.Context, _ []uuid.UUID) error { return nil } +func (m *mockContactStore) GetContactsByMergedID(_ context.Context, _ uuid.UUID) ([]ChannelContact, error) { + return nil, nil +} + +func (m *mockContactStore) upsertCount() int { + m.mu.Lock() + defer m.mu.Unlock() + return len(m.upserts) +} + +// TestContactCollector_SameTenantDedup verifies that repeated calls for the +// same (tenant, channel, sender, thread) only hit the store once. +func TestContactCollector_SameTenantDedup(t *testing.T) { + mock := &mockContactStore{} + c := NewContactCollector(mock, cache.NewInMemoryCache[bool]()) + + tenant := uuid.New() + ctx := WithTenantID(context.Background(), tenant) + + for i := 0; i < 5; i++ { + c.EnsureContact(ctx, "telegram", "tg-main", "user-123", "uid-1", "Alice", "alice", "user", "user", "", "") + } + + if got := mock.upsertCount(); got != 1 { + t.Errorf("same-tenant dedup broken: got %d upserts, want 1", got) + } +} + +// TestContactCollector_CrossTenantIsolation verifies the core bug fix: same +// (channel, sender, thread) in DIFFERENT tenants must produce separate upserts. +// Before the fix, the cache key was missing tenantID so the second tenant's +// upsert was silently skipped, causing a cross-tenant contact leak. +func TestContactCollector_CrossTenantIsolation(t *testing.T) { + mock := &mockContactStore{} + c := NewContactCollector(mock, cache.NewInMemoryCache[bool]()) + + tenantA := uuid.New() + tenantB := uuid.New() + ctxA := WithTenantID(context.Background(), tenantA) + ctxB := WithTenantID(context.Background(), tenantB) + + // Same sender ID "user-123" in both tenants + c.EnsureContact(ctxA, "telegram", "tg-main", "user-123", "uid-1", "Alice", "alice", "user", "user", "", "") + c.EnsureContact(ctxB, "telegram", "tg-main", "user-123", "uid-1", "Alice", "alice", "user", "user", "", "") + + if got := mock.upsertCount(); got != 2 { + t.Errorf("cross-tenant isolation broken: got %d upserts, want 2", got) + } + + // Verify each tenant got its own upsert + mock.mu.Lock() + defer mock.mu.Unlock() + seen := map[uuid.UUID]bool{} + for _, u := range mock.upserts { + seen[u.tenantID] = true + } + if !seen[tenantA] || !seen[tenantB] { + t.Errorf("expected upserts for both tenants, got %+v", seen) + } +} + +// TestContactCollector_ZeroTenantID verifies Desktop edition (no tenant) still +// works with a zero/nil UUID — single-tenant semantics preserved. +func TestContactCollector_ZeroTenantID(t *testing.T) { + mock := &mockContactStore{} + c := NewContactCollector(mock, cache.NewInMemoryCache[bool]()) + + // No tenant in context — Desktop / single-tenant mode + ctx := context.Background() + + c.EnsureContact(ctx, "telegram", "tg", "user-1", "uid", "", "", "user", "user", "", "") + c.EnsureContact(ctx, "telegram", "tg", "user-1", "uid", "", "", "user", "user", "", "") // dup + + if got := mock.upsertCount(); got != 1 { + t.Errorf("zero-tenant dedup broken: got %d upserts, want 1", got) + } +} + +// TestContactCollector_DifferentThreads verifies same sender in different +// threads (same tenant) produces separate upserts. +func TestContactCollector_DifferentThreads(t *testing.T) { + mock := &mockContactStore{} + c := NewContactCollector(mock, cache.NewInMemoryCache[bool]()) + + tenant := uuid.New() + ctx := WithTenantID(context.Background(), tenant) + + c.EnsureContact(ctx, "slack", "ws-1", "user-1", "uid", "", "", "user", "user", "thread-A", "channel") + c.EnsureContact(ctx, "slack", "ws-1", "user-1", "uid", "", "", "user", "user", "thread-B", "channel") + + if got := mock.upsertCount(); got != 2 { + t.Errorf("different-thread isolation broken: got %d upserts, want 2", got) + } +} diff --git a/internal/store/episodic_store.go b/internal/store/episodic_store.go index 44b4aab3..ce390cc6 100644 --- a/internal/store/episodic_store.go +++ b/internal/store/episodic_store.go @@ -10,20 +10,26 @@ import ( // EpisodicSummary represents a Tier 2 episodic memory entry. // Created from session summaries via the consolidation pipeline. type EpisodicSummary struct { - ID uuid.UUID `json:"id" db:"id"` - TenantID uuid.UUID `json:"tenant_id" db:"tenant_id"` - AgentID uuid.UUID `json:"agent_id" db:"agent_id"` - UserID string `json:"user_id" db:"user_id"` // string: chat-based IDs - SessionKey string `json:"session_key" db:"session_key"` - Summary string `json:"summary" db:"summary"` - KeyTopics []string `json:"key_topics" db:"key_topics"` - L0Abstract string `json:"l0_abstract" db:"l0_abstract"` // ~50 tokens, pre-computed - SourceType string `json:"source_type" db:"source_type"` // "session", "v2_daily", "manual" - SourceID string `json:"source_id" db:"source_id"` // dedup key - TurnCount int `json:"turn_count" db:"turn_count"` - TokenCount int `json:"token_count" db:"token_count"` - CreatedAt time.Time `json:"created_at" db:"created_at"` - ExpiresAt *time.Time `json:"expires_at,omitempty" db:"expires_at"` + ID uuid.UUID `json:"id" db:"id"` + TenantID uuid.UUID `json:"tenant_id" db:"tenant_id"` + AgentID uuid.UUID `json:"agent_id" db:"agent_id"` + UserID string `json:"user_id" db:"user_id"` // string: chat-based IDs + SessionKey string `json:"session_key" db:"session_key"` + Summary string `json:"summary" db:"summary"` + KeyTopics []string `json:"key_topics" db:"key_topics"` + L0Abstract string `json:"l0_abstract" db:"l0_abstract"` // ~50 tokens, pre-computed + SourceType string `json:"source_type" db:"source_type"` // "session", "v2_daily", "manual" + SourceID string `json:"source_id" db:"source_id"` // dedup key + TurnCount int `json:"turn_count" db:"turn_count"` + TokenCount int `json:"token_count" db:"token_count"` + CreatedAt time.Time `json:"created_at" db:"created_at"` + ExpiresAt *time.Time `json:"expires_at,omitempty" db:"expires_at"` + + // Phase 10 — dreaming weighted scoring signals. Populated by + // EpisodicStore.RecordRecall; consumed by consolidation.ComputeRecallScore. + RecallCount int `json:"recall_count" db:"recall_count"` + RecallScore float64 `json:"recall_score" db:"recall_score"` // running average of memory_search hit scores + LastRecalledAt *time.Time `json:"last_recalled_at,omitempty" db:"last_recalled_at"` } // EpisodicSearchResult is a search hit with L0 summary. @@ -63,10 +69,18 @@ type EpisodicStore interface { // Promotion lifecycle (used by consolidation pipeline) // ListUnpromoted returns summaries not yet promoted to long-term memory, oldest first. ListUnpromoted(ctx context.Context, agentID, userID string, limit int) ([]EpisodicSummary, error) + // ListUnpromotedScored returns unpromoted summaries ordered by recall_score DESC + // (fallback: created_at ASC). Used by the dreaming worker to prioritise entries + // with stronger recall signals — see internal/consolidation/scoring.go. + ListUnpromotedScored(ctx context.Context, agentID, userID string, limit int) ([]EpisodicSummary, error) // MarkPromoted sets promoted_at=now() for the given IDs. MarkPromoted(ctx context.Context, ids []string) error // CountUnpromoted returns the count of unpromoted summaries for an agent/user. CountUnpromoted(ctx context.Context, agentID, userID string) (int, error) + // RecordRecall updates the per-episode recall signal after a memory_search hit. + // Implementations must increment recall_count, fold `score` into the running + // average stored in recall_score, and set last_recalled_at=NOW(). + RecordRecall(ctx context.Context, id string, score float64) error // Embedding SetEmbeddingProvider(provider EmbeddingProvider) diff --git a/internal/store/evolution_store.go b/internal/store/evolution_store.go index 5e21e189..bf0d8346 100644 --- a/internal/store/evolution_store.go +++ b/internal/store/evolution_store.go @@ -14,7 +14,6 @@ type MetricType string const ( MetricRetrieval MetricType = "retrieval" MetricTool MetricType = "tool" - MetricFeedback MetricType = "feedback" ) // EvolutionMetric is a single recorded metric data point. @@ -58,10 +57,9 @@ type EvolutionMetricsStore interface { type SuggestionType string const ( - SuggestThreshold SuggestionType = "threshold" - SuggestToolOrder SuggestionType = "tool_order" - SuggestSkillAdd SuggestionType = "skill_add" - SuggestMemoryPrune SuggestionType = "memory_prune" + SuggestThreshold SuggestionType = "threshold" + SuggestToolOrder SuggestionType = "tool_order" + SuggestSkillAdd SuggestionType = "skill_add" ) // EvolutionSuggestion is a data-driven suggestion for agent improvement. diff --git a/internal/store/pg/cron_scheduler.go b/internal/store/pg/cron_scheduler.go index 649238b6..e1c6e388 100644 --- a/internal/store/pg/cron_scheduler.go +++ b/internal/store/pg/cron_scheduler.go @@ -3,13 +3,14 @@ package pg import ( "database/sql" "errors" + "fmt" "log/slog" - "sync" "time" "github.com/google/uuid" "github.com/nextlevelbuilder/goclaw/internal/cron" + "github.com/nextlevelbuilder/goclaw/internal/safego" "github.com/nextlevelbuilder/goclaw/internal/store" ) @@ -148,11 +149,22 @@ func (s *PGCronStore) runLoop() { case <-s.stop: return case <-ticker.C: - s.checkAndRunDueJobs() + s.safeCheckAndRunDueJobs() } } } +// safeCheckAndRunDueJobs wraps checkAndRunDueJobs with panic recovery +// so a panic in any check/claim logic doesn't kill the runLoop goroutine. +func (s *PGCronStore) safeCheckAndRunDueJobs() { + defer func() { + if r := recover(); r != nil { + slog.Error("cron: checkAndRunDueJobs panicked — runLoop continues", "panic", fmt.Sprint(r)) + } + }() + s.checkAndRunDueJobs() +} + func (s *PGCronStore) checkAndRunDueJobs() { dueJobs := s.GetDueJobs(time.Now()) if len(dueJobs) == 0 { @@ -178,21 +190,17 @@ func (s *PGCronStore) checkAndRunDueJobs() { return } - // Execute jobs in parallel — scheduler enforces per-session serialization - var wg sync.WaitGroup + // Execute jobs in parallel without blocking the runLoop. + // Previously wg.Wait() blocked here — if any job hung (e.g. LLM timeout, + // agent loop stuck), the entire cron scheduler would stop checking for new + // due jobs. Now each job runs independently; cache is invalidated per-job. for _, job := range claimedJobs { - wg.Add(1) go func(job store.CronJob) { - defer wg.Done() + defer safego.Recover(nil, "component", "cron_job", "job_id", job.ID, "job_name", job.Name) + defer s.InvalidateCache() s.executeOneJob(job, handler, true) }(job) } - wg.Wait() - - // Invalidate cache after job execution changed next_run_at values - s.mu.Lock() - s.cacheLoaded = false - s.mu.Unlock() } // executeOneJob runs a single cron job with retry, logs the result, and updates next_run_at. diff --git a/internal/store/pg/episodic_search.go b/internal/store/pg/episodic_search.go index 70f77add..b9de7aa1 100644 --- a/internal/store/pg/episodic_search.go +++ b/internal/store/pg/episodic_search.go @@ -110,27 +110,15 @@ func mergeEpisodicScores(fts, vec []episodicScored, textWeight, vecWeight float6 return merged } -// scanEpisodic scans a single row into EpisodicSummary. +// scanEpisodic scans a single row into EpisodicSummary. Column order matches +// the SELECT list in PGEpisodicStore.Get (17 columns incl. recall signals). func scanEpisodic(row *sql.Row) (*store.EpisodicSummary, error) { var ep store.EpisodicSummary var topics pq.StringArray err := row.Scan(&ep.ID, &ep.TenantID, &ep.AgentID, &ep.UserID, &ep.SessionKey, &ep.Summary, &topics, &ep.TurnCount, &ep.TokenCount, - &ep.L0Abstract, &ep.SourceID, &ep.SourceType, &ep.CreatedAt, &ep.ExpiresAt) - if err != nil { - return nil, err - } - ep.KeyTopics = []string(topics) - return &ep, nil -} - -// scanEpisodicRow scans from sql.Rows (same fields as scanEpisodic). -func scanEpisodicRow(rows *sql.Rows) (*store.EpisodicSummary, error) { - var ep store.EpisodicSummary - var topics pq.StringArray - err := rows.Scan(&ep.ID, &ep.TenantID, &ep.AgentID, &ep.UserID, &ep.SessionKey, - &ep.Summary, &topics, &ep.TurnCount, &ep.TokenCount, - &ep.L0Abstract, &ep.SourceID, &ep.SourceType, &ep.CreatedAt, &ep.ExpiresAt) + &ep.L0Abstract, &ep.SourceID, &ep.SourceType, &ep.CreatedAt, &ep.ExpiresAt, + &ep.RecallCount, &ep.RecallScore, &ep.LastRecalledAt) if err != nil { return nil, err } diff --git a/internal/store/pg/episodic_summaries.go b/internal/store/pg/episodic_summaries.go index e7705094..3afacf70 100644 --- a/internal/store/pg/episodic_summaries.go +++ b/internal/store/pg/episodic_summaries.go @@ -68,7 +68,7 @@ func (s *PGEpisodicStore) Get(ctx context.Context, id string) (*store.EpisodicSu row := s.db.QueryRowContext(ctx, ` SELECT id, tenant_id, agent_id, user_id, session_key, summary, key_topics, turn_count, token_count, l0_abstract, source_id, source_type, - created_at, expires_at + created_at, expires_at, recall_count, recall_score, last_recalled_at FROM episodic_summaries WHERE id = $1 AND tenant_id = $2`, id, store.TenantIDFromContext(ctx)) return scanEpisodic(row) @@ -94,7 +94,8 @@ func (s *PGEpisodicStore) List(ctx context.Context, agentID, userID string, limi if userID != "" { q = `SELECT id, tenant_id, agent_id, user_id, session_key, summary, key_topics, turn_count, token_count, l0_abstract, source_id, source_type, - created_at, expires_at + created_at, expires_at, + recall_count, recall_score, last_recalled_at FROM episodic_summaries WHERE agent_id = $1 AND user_id = $2 AND tenant_id = $5 ORDER BY created_at DESC LIMIT $3 OFFSET $4` @@ -102,7 +103,8 @@ func (s *PGEpisodicStore) List(ctx context.Context, agentID, userID string, limi } else { q = `SELECT id, tenant_id, agent_id, user_id, session_key, summary, key_topics, turn_count, token_count, l0_abstract, source_id, source_type, - created_at, expires_at + created_at, expires_at, + recall_count, recall_score, last_recalled_at FROM episodic_summaries WHERE agent_id = $1 AND tenant_id = $4 ORDER BY created_at DESC LIMIT $2 OFFSET $3` @@ -194,19 +196,34 @@ func (s *PGEpisodicStore) PruneExpired(ctx context.Context) (int, error) { // ListUnpromoted returns episodic summaries not yet promoted to long-term memory, oldest first. // Scoped to agent/user within the caller's tenant. func (s *PGEpisodicStore) ListUnpromoted(ctx context.Context, agentID, userID string, limit int) ([]store.EpisodicSummary, error) { + return s.listUnpromoted(ctx, agentID, userID, limit, "created_at ASC") +} + +// ListUnpromotedScored returns unpromoted episodic summaries ordered by +// recall_score DESC (ties broken by created_at ASC so older entries with the +// same score synthesise first). Backed by `idx_episodic_recall_unpromoted`. +func (s *PGEpisodicStore) ListUnpromotedScored(ctx context.Context, agentID, userID string, limit int) ([]store.EpisodicSummary, error) { + return s.listUnpromoted(ctx, agentID, userID, limit, "recall_score DESC, created_at ASC") +} + +// listUnpromoted shares the query shape between the two ListUnpromoted* +// variants. `orderBy` is a static literal supplied by the caller — it is +// NEVER derived from user input, so the concatenation below is safe. +func (s *PGEpisodicStore) listUnpromoted(ctx context.Context, agentID, userID string, limit int, orderBy string) ([]store.EpisodicSummary, error) { if limit <= 0 { limit = 20 } tenantID := store.TenantIDFromContext(ctx) var rows []episodicSummaryRow - err := pkgSqlxDB.SelectContext(ctx, &rows, ` + query := ` SELECT id, tenant_id, agent_id, user_id, session_key, summary, key_topics, turn_count, token_count, l0_abstract, source_id, source_type, - created_at, expires_at + created_at, expires_at, + recall_count, recall_score, last_recalled_at FROM episodic_summaries WHERE agent_id = $1 AND user_id = $2 AND tenant_id = $3 AND promoted_at IS NULL - ORDER BY created_at ASC LIMIT $4`, - agentID, userID, tenantID, limit) + ORDER BY ` + orderBy + ` LIMIT $4` + err := pkgSqlxDB.SelectContext(ctx, &rows, query, agentID, userID, tenantID, limit) if err != nil { return nil, fmt.Errorf("episodic list_unpromoted: %w", err) } @@ -217,6 +234,37 @@ func (s *PGEpisodicStore) ListUnpromoted(ctx context.Context, agentID, userID st return results, nil } +// RecordRecall increments recall_count, folds `score` into the running +// average stored in recall_score, and sets last_recalled_at=NOW(). Uses a +// single UPDATE so the row is rewritten atomically. +// +// The running-average expression reads the OLD values of recall_count / +// recall_score (Postgres evaluates the SET clause against the pre-update +// tuple), so the result is `(oldAvg*oldCount + newScore) / (oldCount+1)`. +func (s *PGEpisodicStore) RecordRecall(ctx context.Context, id string, score float64) error { + if id == "" { + return nil + } + // Clamp inputs so bad data from callers cannot corrupt the running average. + if score < 0 { + score = 0 + } else if score > 1 { + score = 1 + } + tenantID := store.TenantIDFromContext(ctx) + _, err := s.db.ExecContext(ctx, ` + UPDATE episodic_summaries + SET recall_count = recall_count + 1, + recall_score = (recall_score * recall_count + $1) / (recall_count + 1), + last_recalled_at = NOW() + WHERE id = $2 AND tenant_id = $3`, + score, id, tenantID) + if err != nil { + return fmt.Errorf("episodic record_recall: %w", err) + } + return nil +} + // MarkPromoted sets promoted_at=now() for the given episodic summary IDs. // Scoped to the caller's tenant for safety. func (s *PGEpisodicStore) MarkPromoted(ctx context.Context, ids []string) error { diff --git a/internal/store/pg/memory_scan_rows.go b/internal/store/pg/memory_scan_rows.go index 084460d7..80f0ccfa 100644 --- a/internal/store/pg/memory_scan_rows.go +++ b/internal/store/pg/memory_scan_rows.go @@ -102,34 +102,40 @@ func (r *scoredChunkRow) toScoredChunk() scoredChunk { // episodicSummaryRow is an sqlx scan struct for episodic_summaries SELECT queries. // Handles TEXT[] key_topics via pq.StringArray. type episodicSummaryRow struct { - ID string `db:"id"` - TenantID string `db:"tenant_id"` - AgentID string `db:"agent_id"` - UserID string `db:"user_id"` - SessionKey string `db:"session_key"` - Summary string `db:"summary"` - KeyTopics pq.StringArray `db:"key_topics"` - TurnCount int `db:"turn_count"` - TokenCount int `db:"token_count"` - L0Abstract string `db:"l0_abstract"` - SourceID string `db:"source_id"` - SourceType string `db:"source_type"` - CreatedAt time.Time `db:"created_at"` - ExpiresAt *time.Time `db:"expires_at"` + ID string `db:"id"` + TenantID string `db:"tenant_id"` + AgentID string `db:"agent_id"` + UserID string `db:"user_id"` + SessionKey string `db:"session_key"` + Summary string `db:"summary"` + KeyTopics pq.StringArray `db:"key_topics"` + TurnCount int `db:"turn_count"` + TokenCount int `db:"token_count"` + L0Abstract string `db:"l0_abstract"` + SourceID string `db:"source_id"` + SourceType string `db:"source_type"` + CreatedAt time.Time `db:"created_at"` + ExpiresAt *time.Time `db:"expires_at"` + RecallCount int `db:"recall_count"` + RecallScore float64 `db:"recall_score"` + LastRecalledAt *time.Time `db:"last_recalled_at"` } func (r *episodicSummaryRow) toEpisodicSummary() store.EpisodicSummary { ep := store.EpisodicSummary{ - UserID: r.UserID, - SessionKey: r.SessionKey, - Summary: r.Summary, - TurnCount: r.TurnCount, - TokenCount: r.TokenCount, - L0Abstract: r.L0Abstract, - SourceID: r.SourceID, - SourceType: r.SourceType, - CreatedAt: r.CreatedAt, - ExpiresAt: r.ExpiresAt, + UserID: r.UserID, + SessionKey: r.SessionKey, + Summary: r.Summary, + TurnCount: r.TurnCount, + TokenCount: r.TokenCount, + L0Abstract: r.L0Abstract, + SourceID: r.SourceID, + SourceType: r.SourceType, + CreatedAt: r.CreatedAt, + ExpiresAt: r.ExpiresAt, + RecallCount: r.RecallCount, + RecallScore: r.RecallScore, + LastRecalledAt: r.LastRecalledAt, } _ = ep.ID.Scan(r.ID) _ = ep.TenantID.Scan(r.TenantID) diff --git a/internal/store/pg/tenant_store.go b/internal/store/pg/tenant_store.go index e837166a..e5b72906 100644 --- a/internal/store/pg/tenant_store.go +++ b/internal/store/pg/tenant_store.go @@ -8,6 +8,7 @@ import ( "time" "github.com/google/uuid" + "github.com/lib/pq" "github.com/nextlevelbuilder/goclaw/internal/store" ) @@ -80,6 +81,27 @@ func (s *PGTenantStore) ListTenants(ctx context.Context) ([]store.TenantData, er return tenants, nil } +func (s *PGTenantStore) GetTenantsByIDs(ctx context.Context, ids []uuid.UUID) ([]store.TenantData, error) { + if len(ids) == 0 { + return nil, nil + } + // Chunk by 500 to stay well within PG param limits. + const chunkSize = 500 + var all []store.TenantData + for start := 0; start < len(ids); start += chunkSize { + end := min(start+chunkSize, len(ids)) + var chunk []store.TenantData + if err := pkgSqlxDB.SelectContext(ctx, &chunk, + `SELECT id, name, slug, status, settings, created_at, updated_at + FROM tenants WHERE id = ANY($1)`, + pq.Array(ids[start:end])); err != nil { + return nil, err + } + all = append(all, chunk...) + } + return all, nil +} + func (s *PGTenantStore) UpdateTenant(ctx context.Context, id uuid.UUID, updates map[string]any) error { return execMapUpdate(ctx, s.db, "tenants", id, updates) } diff --git a/internal/store/pg/vault_documents.go b/internal/store/pg/vault_documents.go index 7784766c..66f7bb92 100644 --- a/internal/store/pg/vault_documents.go +++ b/internal/store/pg/vault_documents.go @@ -54,10 +54,19 @@ func (s *PGVaultStore) SetEmbeddingProvider(provider store.EmbeddingProvider) { func (s *PGVaultStore) Close() error { return nil } +// optAgentUUID converts a nullable *string agent_id to *uuid.UUID for SQL. +func optAgentUUID(agentID *string) *uuid.UUID { + if agentID == nil || *agentID == "" { + return nil + } + u := mustParseUUID(*agentID) + return &u +} + // UpsertDocument inserts or updates a vault document. func (s *PGVaultStore) UpsertDocument(ctx context.Context, doc *store.VaultDocument) error { tid := mustParseUUID(doc.TenantID) - aid := mustParseUUID(doc.AgentID) + aid := optAgentUUID(doc.AgentID) now := time.Now().UTC() meta, err := json.Marshal(doc.Metadata) @@ -91,7 +100,7 @@ func (s *PGVaultStore) UpsertDocument(ctx context.Context, doc *store.VaultDocum INSERT INTO vault_documents (id, tenant_id, agent_id, team_id, scope, custom_scope, path, title, doc_type, content_hash, summary, embedding, metadata, created_at, updated_at) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $14) - ON CONFLICT (agent_id, COALESCE(team_id, '00000000-0000-0000-0000-000000000000'), scope, path) DO UPDATE SET + ON CONFLICT (tenant_id, COALESCE(agent_id, '00000000-0000-0000-0000-000000000000'::uuid), COALESCE(team_id, '00000000-0000-0000-0000-000000000000'::uuid), scope, path) DO UPDATE SET title = EXCLUDED.title, doc_type = EXCLUDED.doc_type, content_hash = EXCLUDED.content_hash, @@ -112,15 +121,21 @@ func (s *PGVaultStore) UpsertDocument(ctx context.Context, doc *store.VaultDocum } // GetDocument retrieves a vault document by tenant, agent, and path. +// Empty agentID means no agent filter (match any agent). // Team scoping via RunContext: present+TeamID → filter; present+empty → personal; nil → any match. func (s *PGVaultStore) GetDocument(ctx context.Context, tenantID, agentID, path string) (*store.VaultDocument, error) { tid := mustParseUUID(tenantID) - aid := mustParseUUID(agentID) q := `SELECT id, tenant_id, agent_id, team_id, scope, custom_scope, path, title, doc_type, content_hash, summary, metadata, created_at, updated_at - FROM vault_documents WHERE tenant_id = $1 AND agent_id = $2 AND path = $3` - args := []any{tid, aid, path} - p := 4 + FROM vault_documents WHERE tenant_id = $1 AND path = $2` + args := []any{tid, path} + p := 3 + + if agentID != "" { + q += fmt.Sprintf(" AND agent_id = $%d", p) + args = append(args, mustParseUUID(agentID)) + p++ + } if rc := store.RunContextFromCtx(ctx); rc != nil { if rc.TeamID != "" { @@ -161,15 +176,72 @@ func (s *PGVaultStore) GetDocumentByID(ctx context.Context, tenantID, id string) return &doc, nil } +// GetDocumentsByIDs returns documents matching the given IDs with tenant isolation. +// Chunks by 500 to stay within PG param limits. +func (s *PGVaultStore) GetDocumentsByIDs(ctx context.Context, tenantID string, docIDs []string) ([]store.VaultDocument, error) { + if len(docIDs) == 0 { + return nil, nil + } + tid := mustParseUUID(tenantID) + const chunkSize = 500 + var all []store.VaultDocument + for start := 0; start < len(docIDs); start += chunkSize { + end := min(start+chunkSize, len(docIDs)) + var scanned []vaultDocRow + if err := pkgSqlxDB.SelectContext(ctx, &scanned, + `SELECT id, tenant_id, agent_id, team_id, scope, custom_scope, path, title, doc_type, content_hash, summary, metadata, created_at, updated_at + FROM vault_documents WHERE id = ANY($1) AND tenant_id = $2`, + pqStringArray(docIDs[start:end]), tid); err != nil { + return nil, err + } + for i := range scanned { + all = append(all, scanned[i].toVaultDocument()) + } + } + return all, nil +} + +// GetDocumentByBasename finds a document by path basename (case-insensitive). +// Uses the stored generated column path_basename + index for fast lookup. +func (s *PGVaultStore) GetDocumentByBasename(ctx context.Context, tenantID, agentID, basename string) (*store.VaultDocument, error) { + tid := mustParseUUID(tenantID) + q := `SELECT id, tenant_id, agent_id, team_id, scope, custom_scope, path, title, doc_type, content_hash, summary, metadata, created_at, updated_at + FROM vault_documents + WHERE tenant_id = $1 AND path_basename = lower($2)` + args := []any{tid, basename} + p := 3 + if agentID != "" { + q += fmt.Sprintf(" AND agent_id = $%d", p) + args = append(args, mustParseUUID(agentID)) + } + q += " LIMIT 1" + var row vaultDocRow + err := s.db.QueryRowContext(ctx, q, args...).Scan( + &row.ID, &row.TenantID, &row.AgentID, &row.TeamID, &row.Scope, &row.CustomScope, + &row.Path, &row.Title, &row.DocType, &row.ContentHash, &row.Summary, + &row.MetaJSON, &row.CreatedAt, &row.UpdatedAt) + if err != nil { + return nil, err + } + doc := row.toVaultDocument() + return &doc, nil +} + // DeleteDocument removes a vault document by tenant, agent, and path. +// Empty agentID means no agent filter. // Team scoping via RunContext (same rules as GetDocument). func (s *PGVaultStore) DeleteDocument(ctx context.Context, tenantID, agentID, path string) error { tid := mustParseUUID(tenantID) - aid := mustParseUUID(agentID) - q := `DELETE FROM vault_documents WHERE tenant_id = $1 AND agent_id = $2 AND path = $3` - args := []any{tid, aid, path} - p := 4 + q := `DELETE FROM vault_documents WHERE tenant_id = $1 AND path = $2` + args := []any{tid, path} + p := 3 + + if agentID != "" { + q += fmt.Sprintf(" AND agent_id = $%d", p) + args = append(args, mustParseUUID(agentID)) + p++ + } if rc := store.RunContextFromCtx(ctx); rc != nil { if rc.TeamID != "" { @@ -287,7 +359,7 @@ func (s *PGVaultStore) UpdateHash(ctx context.Context, tenantID, id, newHash str // Search performs hybrid FTS + vector search on vault_documents. func (s *PGVaultStore) Search(ctx context.Context, opts store.VaultSearchOptions) ([]store.VaultSearchResult, error) { tid := mustParseUUID(opts.TenantID) - aid := mustParseUUID(opts.AgentID) + aid := optAgentUUID(&opts.AgentID) // empty string → nil → no agent filter // Build team filter for search sub-queries. tf := buildSearchTeamFilter(opts.TeamID, opts.TeamIDs) @@ -381,13 +453,19 @@ func (tf searchTeamFilter) append(q string, args []any, p int) (string, []any, i return q, args, p } -func (s *PGVaultStore) ftsSearch(ctx context.Context, query string, tenantID, agentID uuid.UUID, tf searchTeamFilter, scope string, docTypes []string, limit int) ([]store.VaultSearchResult, error) { +func (s *PGVaultStore) ftsSearch(ctx context.Context, query string, tenantID uuid.UUID, agentID *uuid.UUID, tf searchTeamFilter, scope string, docTypes []string, limit int) ([]store.VaultSearchResult, error) { q := `SELECT id, tenant_id, agent_id, team_id, scope, custom_scope, path, title, doc_type, content_hash, summary, metadata, created_at, updated_at, ts_rank(tsv, plainto_tsquery('simple', $1)) AS score FROM vault_documents - WHERE tenant_id = $2 AND agent_id = $3 AND tsv @@ plainto_tsquery('simple', $1)` - args := []any{query, tenantID, agentID} - p := 4 + WHERE tenant_id = $2 AND tsv @@ plainto_tsquery('simple', $1)` + args := []any{query, tenantID} + p := 3 + + if agentID != nil { + q += fmt.Sprintf(" AND agent_id = $%d", p) + args = append(args, *agentID) + p++ + } q, args, p = tf.append(q, args, p) @@ -412,14 +490,20 @@ func (s *PGVaultStore) ftsSearch(ctx context.Context, query string, tenantID, ag return vaultSearchRowsToResults(scanned, "vault"), nil } -func (s *PGVaultStore) vectorSearch(ctx context.Context, embedding []float32, tenantID, agentID uuid.UUID, tf searchTeamFilter, scope string, docTypes []string, limit int) ([]store.VaultSearchResult, error) { +func (s *PGVaultStore) vectorSearch(ctx context.Context, embedding []float32, tenantID uuid.UUID, agentID *uuid.UUID, tf searchTeamFilter, scope string, docTypes []string, limit int) ([]store.VaultSearchResult, error) { vecStr := vectorToString(embedding) q := `SELECT id, tenant_id, agent_id, team_id, scope, custom_scope, path, title, doc_type, content_hash, summary, metadata, created_at, updated_at, 1 - (embedding <=> $1) AS score FROM vault_documents - WHERE tenant_id = $2 AND agent_id = $3 AND embedding IS NOT NULL` - args := []any{vecStr, tenantID, agentID} - p := 4 + WHERE tenant_id = $2 AND embedding IS NOT NULL` + args := []any{vecStr, tenantID} + p := 3 + + if agentID != nil { + q += fmt.Sprintf(" AND agent_id = $%d", p) + args = append(args, *agentID) + p++ + } q, args, p = tf.append(q, args, p) diff --git a/internal/store/pg/vault_documents_enrichment.go b/internal/store/pg/vault_documents_enrichment.go index 9d448583..fbbbf1b0 100644 --- a/internal/store/pg/vault_documents_enrichment.go +++ b/internal/store/pg/vault_documents_enrichment.go @@ -44,9 +44,10 @@ func (s *PGVaultStore) UpdateSummaryAndReembed(ctx context.Context, tenantID, do // FindSimilarDocs finds documents with similar embeddings to the given docID. // Returns top-N neighbors excluding the source doc itself. +// Empty agentID means no agent filter. func (s *PGVaultStore) FindSimilarDocs(ctx context.Context, tenantID, agentID, docID string, limit int) ([]store.VaultSearchResult, error) { tid := mustParseUUID(tenantID) - aid := mustParseUUID(agentID) + aid := optAgentUUID(&agentID) did := mustParseUUID(docID) // Fetch source embedding. @@ -63,12 +64,20 @@ func (s *PGVaultStore) FindSimilarDocs(ctx context.Context, tenantID, agentID, d content_hash, summary, metadata, created_at, updated_at, 1 - (embedding <=> $1::vector) AS score FROM vault_documents - WHERE tenant_id = $2 AND agent_id = $3 AND id != $4 AND embedding IS NOT NULL - ORDER BY embedding <=> $1::vector - LIMIT $5` + WHERE tenant_id = $2 AND id != $3 AND embedding IS NOT NULL` + args := []any{*embStr, tid, did} + p := 4 + + if aid != nil { + q += fmt.Sprintf(" AND agent_id = $%d", p) + args = append(args, *aid) + p++ + } + q += fmt.Sprintf(" ORDER BY embedding <=> $1::vector LIMIT $%d", p) + args = append(args, limit) var scanned []vaultSearchRow - if err := pkgSqlxDB.SelectContext(ctx, &scanned, q, *embStr, tid, aid, did, limit); err != nil { + if err := pkgSqlxDB.SelectContext(ctx, &scanned, q, args...); err != nil { return nil, fmt.Errorf("vault.find_similar: %w", err) } return vaultSearchRowsToResults(scanned, "vault"), nil diff --git a/internal/store/pg/vault_links.go b/internal/store/pg/vault_links.go index a7ab7c13..54903073 100644 --- a/internal/store/pg/vault_links.go +++ b/internal/store/pg/vault_links.go @@ -4,12 +4,100 @@ import ( "context" "database/sql" "fmt" + "strings" "time" "github.com/google/uuid" "github.com/nextlevelbuilder/goclaw/internal/store" ) +// CreateLinks batch-inserts vault links with tenant validation. +// Validates same-tenant boundary for all links, then multi-row INSERT. +func (s *PGVaultStore) CreateLinks(ctx context.Context, links []store.VaultLink) error { + if len(links) == 0 { + return nil + } + + // Collect all unique doc IDs for tenant validation. + docIDSet := make(map[string]struct{}) + for _, l := range links { + docIDSet[l.FromDocID] = struct{}{} + docIDSet[l.ToDocID] = struct{}{} + } + docIDs := make([]string, 0, len(docIDSet)) + for id := range docIDSet { + docIDs = append(docIDs, id) + } + + // Batch-fetch tenant_id for all referenced docs. + type docMeta struct { + tenantID uuid.UUID + teamID *uuid.UUID + } + docMap := make(map[string]docMeta, len(docIDs)) + rows, err := s.db.QueryContext(ctx, + `SELECT id, tenant_id, team_id FROM vault_documents WHERE id = ANY($1)`, + pqStringArray(docIDs)) + if err != nil { + return fmt.Errorf("vault batch link: fetch docs: %w", err) + } + defer rows.Close() + for rows.Next() { + var id, tid uuid.UUID + var teamID *uuid.UUID + if err := rows.Scan(&id, &tid, &teamID); err != nil { + return fmt.Errorf("vault batch link: scan doc: %w", err) + } + docMap[id.String()] = docMeta{tenantID: tid, teamID: teamID} + } + if err := rows.Err(); err != nil { + return fmt.Errorf("vault batch link: rows err: %w", err) + } + + // Validate and build multi-row INSERT (chunk by 500). + const chunkSize = 500 + var valid []store.VaultLink + for _, l := range links { + from, fromOK := docMap[l.FromDocID] + to, toOK := docMap[l.ToDocID] + if !fromOK || !toOK { + continue // skip if doc not found + } + if from.tenantID != to.tenantID { + continue // cross-tenant + } + if from.teamID != nil && to.teamID != nil && *from.teamID != *to.teamID { + continue // cross-team + } + valid = append(valid, l) + } + + for start := 0; start < len(valid); start += chunkSize { + end := min(start+chunkSize, len(valid)) + chunk := valid[start:end] + const paramsPerRow = 6 // id, from_doc_id, to_doc_id, link_type, context, created_at + args := make([]any, 0, len(chunk)*paramsPerRow) + ph := make([]string, 0, len(chunk)) + now := time.Now().UTC() + for i, l := range chunk { + b := i * paramsPerRow + ph = append(ph, fmt.Sprintf("($%d,$%d,$%d,$%d,$%d,$%d)", b+1, b+2, b+3, b+4, b+5, b+6)) + args = append(args, + uuid.Must(uuid.NewV7()), + mustParseUUID(l.FromDocID), mustParseUUID(l.ToDocID), + l.LinkType, l.Context, now, + ) + } + q := `INSERT INTO vault_links (id, from_doc_id, to_doc_id, link_type, context, created_at) + VALUES ` + strings.Join(ph, ",") + ` + ON CONFLICT (from_doc_id, to_doc_id, link_type) DO UPDATE SET context = EXCLUDED.context` + if _, err := s.db.ExecContext(ctx, q, args...); err != nil { + return fmt.Errorf("vault batch create links: %w", err) + } + } + return nil +} + // CreateLink inserts a vault link, updating context on conflict. // Validates same-tenant + same-team boundary before insert. func (s *PGVaultStore) CreateLink(ctx context.Context, link *store.VaultLink) error { @@ -87,6 +175,25 @@ func (s *PGVaultStore) GetOutLinks(ctx context.Context, tenantID, docID string) return scanVaultLinks(rows) } +// GetOutLinksBatch returns all outlinks for multiple doc IDs in a single query. +func (s *PGVaultStore) GetOutLinksBatch(ctx context.Context, tenantID string, docIDs []string) ([]store.VaultLink, error) { + if len(docIDs) == 0 { + return nil, nil + } + tid := mustParseUUID(tenantID) + rows, err := s.db.QueryContext(ctx, ` + SELECT vl.id, vl.from_doc_id, vl.to_doc_id, vl.link_type, vl.context, vl.created_at + FROM vault_links vl + JOIN vault_documents vd ON vl.from_doc_id = vd.id + WHERE vl.from_doc_id = ANY($1) AND vd.tenant_id = $2 + ORDER BY vl.created_at`, pqStringArray(docIDs), tid) + if err != nil { + return nil, err + } + defer rows.Close() + return scanVaultLinks(rows) +} + // GetBacklinks returns enriched backlinks pointing to a document (single JOIN, LIMIT 100). func (s *PGVaultStore) GetBacklinks(ctx context.Context, tenantID, docID string) ([]store.VaultBacklink, error) { did := mustParseUUID(docID) @@ -132,6 +239,43 @@ func (s *PGVaultStore) DeleteDocLinks(ctx context.Context, tenantID, docID strin return err } +// DeleteDocLinksByType removes outbound links of a specific type from a document. +func (s *PGVaultStore) DeleteDocLinksByType(ctx context.Context, tenantID, docID, linkType string) error { + uid := mustParseUUID(docID) + tid := mustParseUUID(tenantID) + _, err := s.db.ExecContext(ctx, ` + DELETE FROM vault_links vl + USING vault_documents vd + WHERE vl.from_doc_id = $1 + AND vd.id = vl.from_doc_id AND vd.tenant_id = $2 + AND vl.link_type = $3`, uid, tid, linkType) + return err +} + +// DeleteDocLinksByTypes removes outbound links matching any of the given types from a document. +func (s *PGVaultStore) DeleteDocLinksByTypes(ctx context.Context, tenantID, docID string, types []string) error { + if len(types) == 0 { + return nil + } + uid := mustParseUUID(docID) + tid := mustParseUUID(tenantID) + // Build IN clause with positional params: $3, $4, ... + params := []any{uid, tid} + placeholders := make([]string, len(types)) + for i, t := range types { + params = append(params, t) + placeholders[i] = fmt.Sprintf("$%d", i+3) + } + q := fmt.Sprintf(` + DELETE FROM vault_links vl + USING vault_documents vd + WHERE vl.from_doc_id = $1 + AND vd.id = vl.from_doc_id AND vd.tenant_id = $2 + AND vl.link_type IN (%s)`, strings.Join(placeholders, ",")) + _, err := s.db.ExecContext(ctx, q, params...) + return err +} + func scanVaultLinks(rows *sql.Rows) ([]store.VaultLink, error) { var links []store.VaultLink for rows.Next() { diff --git a/internal/store/pg/vault_scan_rows.go b/internal/store/pg/vault_scan_rows.go index 1748729c..96ea271c 100644 --- a/internal/store/pg/vault_scan_rows.go +++ b/internal/store/pg/vault_scan_rows.go @@ -14,7 +14,7 @@ import ( type vaultDocRow struct { ID uuid.UUID `db:"id"` TenantID uuid.UUID `db:"tenant_id"` - AgentID uuid.UUID `db:"agent_id"` + AgentID *uuid.UUID `db:"agent_id"` TeamID *uuid.UUID `db:"team_id"` Scope string `db:"scope"` CustomScope *string `db:"custom_scope"` @@ -33,7 +33,6 @@ func (r *vaultDocRow) toVaultDocument() store.VaultDocument { doc := store.VaultDocument{ ID: r.ID.String(), TenantID: r.TenantID.String(), - AgentID: r.AgentID.String(), Scope: r.Scope, CustomScope: r.CustomScope, Path: r.Path, @@ -44,6 +43,10 @@ func (r *vaultDocRow) toVaultDocument() store.VaultDocument { CreatedAt: r.CreatedAt, UpdatedAt: r.UpdatedAt, } + if r.AgentID != nil { + s := r.AgentID.String() + doc.AgentID = &s + } if r.TeamID != nil { s := r.TeamID.String() doc.TeamID = &s diff --git a/internal/store/sqlitestore/cron_crud_test.go b/internal/store/sqlitestore/cron_crud_test.go index c9b1c8b9..8423df1a 100644 --- a/internal/store/sqlitestore/cron_crud_test.go +++ b/internal/store/sqlitestore/cron_crud_test.go @@ -230,7 +230,7 @@ func TestSQLiteCronStore_ExecuteOneJob_DoesNotRestoreNextRunAfterDisable(t *test t.Fatalf("disable during run error: %v", err) } return &store.CronJobResult{Content: "ok"}, nil - }) + }, false) current := mustRawJob(t, db, jobUUID) if current.enabled { diff --git a/internal/store/sqlitestore/cron_scheduler.go b/internal/store/sqlitestore/cron_scheduler.go index 3b92d78d..26302358 100644 --- a/internal/store/sqlitestore/cron_scheduler.go +++ b/internal/store/sqlitestore/cron_scheduler.go @@ -5,13 +5,14 @@ package sqlitestore import ( "database/sql" "errors" + "fmt" "log/slog" - "sync" "time" "github.com/google/uuid" "github.com/nextlevelbuilder/goclaw/internal/cron" + "github.com/nextlevelbuilder/goclaw/internal/safego" "github.com/nextlevelbuilder/goclaw/internal/store" ) @@ -154,11 +155,22 @@ func (s *SQLiteCronStore) runLoop() { case <-s.stop: return case <-ticker.C: - s.checkAndRunDueJobs() + s.safeCheckAndRunDueJobs() } } } +// safeCheckAndRunDueJobs wraps checkAndRunDueJobs with panic recovery +// so a panic in any check/claim logic doesn't kill the runLoop goroutine. +func (s *SQLiteCronStore) safeCheckAndRunDueJobs() { + defer func() { + if r := recover(); r != nil { + slog.Error("cron: checkAndRunDueJobs panicked — runLoop continues", "panic", fmt.Sprint(r)) + } + }() + s.checkAndRunDueJobs() +} + func (s *SQLiteCronStore) checkAndRunDueJobs() { dueJobs := s.GetDueJobs(time.Now()) if len(dueJobs) == 0 { @@ -184,20 +196,17 @@ func (s *SQLiteCronStore) checkAndRunDueJobs() { return } - // Execute jobs in parallel — scheduler enforces per-session serialization. - var wg sync.WaitGroup + // Execute jobs in parallel without blocking the runLoop. + // Previously wg.Wait() blocked here — if any job hung (e.g. LLM timeout, + // agent loop stuck), the entire cron scheduler would stop checking for new + // due jobs. Now each job runs independently; cache is invalidated per-job. for _, job := range claimedJobs { - wg.Add(1) go func(job store.CronJob) { - defer wg.Done() + defer safego.Recover(nil, "component", "cron_job", "job_id", job.ID, "job_name", job.Name) + defer s.InvalidateCache() s.executeOneJob(job, handler, true) }(job) } - wg.Wait() - - s.mu.Lock() - s.cacheLoaded = false - s.mu.Unlock() } // executeOneJob runs a claimed job. When reloadClaimed is true (scheduler path), diff --git a/internal/store/sqlitestore/episodic.go b/internal/store/sqlitestore/episodic.go index 2456ff08..10fc16fa 100644 --- a/internal/store/sqlitestore/episodic.go +++ b/internal/store/sqlitestore/episodic.go @@ -69,7 +69,7 @@ func (s *SQLiteEpisodicStore) Get(ctx context.Context, id string) (*store.Episod row := s.db.QueryRowContext(ctx, ` SELECT id, tenant_id, agent_id, user_id, session_key, summary, key_topics, turn_count, token_count, l0_abstract, source_id, source_type, - created_at, expires_at + created_at, expires_at, recall_count, recall_score, last_recalled_at FROM episodic_summaries WHERE id = ? AND tenant_id = ?`, id, tenantID.String()) return scanSQLiteEpisodic(row) @@ -96,7 +96,7 @@ func (s *SQLiteEpisodicStore) List(ctx context.Context, agentID, userID string, if userID != "" { q = `SELECT id, tenant_id, agent_id, user_id, session_key, summary, key_topics, turn_count, token_count, l0_abstract, source_id, source_type, - created_at, expires_at + created_at, expires_at, recall_count, recall_score, last_recalled_at FROM episodic_summaries WHERE agent_id = ? AND user_id = ? AND tenant_id = ? ORDER BY created_at DESC LIMIT ? OFFSET ?` @@ -104,7 +104,7 @@ func (s *SQLiteEpisodicStore) List(ctx context.Context, agentID, userID string, } else { q = `SELECT id, tenant_id, agent_id, user_id, session_key, summary, key_topics, turn_count, token_count, l0_abstract, source_id, source_type, - created_at, expires_at + created_at, expires_at, recall_count, recall_score, last_recalled_at FROM episodic_summaries WHERE agent_id = ? AND tenant_id = ? ORDER BY created_at DESC LIMIT ? OFFSET ?` @@ -148,18 +148,33 @@ func (s *SQLiteEpisodicStore) PruneExpired(ctx context.Context) (int, error) { // ListUnpromoted returns summaries not yet promoted, oldest first. func (s *SQLiteEpisodicStore) ListUnpromoted(ctx context.Context, agentID, userID string, limit int) ([]store.EpisodicSummary, error) { + return s.listUnpromoted(ctx, agentID, userID, limit, "created_at ASC") +} + +// ListUnpromotedScored returns unpromoted summaries ordered by recall_score DESC +// (ties broken by created_at ASC). Backed by idx_episodic_recall_unpromoted +// (schema.sql). Mirrors PGEpisodicStore.ListUnpromotedScored. +func (s *SQLiteEpisodicStore) ListUnpromotedScored(ctx context.Context, agentID, userID string, limit int) ([]store.EpisodicSummary, error) { + return s.listUnpromoted(ctx, agentID, userID, limit, "recall_score DESC, created_at ASC") +} + +// listUnpromoted shares the query shape between the two ListUnpromoted* +// variants. `orderBy` is a static literal from the caller, never derived +// from user input, so the concatenation below is safe. +func (s *SQLiteEpisodicStore) listUnpromoted(ctx context.Context, agentID, userID string, limit int, orderBy string) ([]store.EpisodicSummary, error) { if limit <= 0 { limit = 20 } tenantID := tenantIDForInsert(ctx) - rows, err := s.db.QueryContext(ctx, ` + query := ` SELECT id, tenant_id, agent_id, user_id, session_key, summary, key_topics, turn_count, token_count, l0_abstract, source_id, source_type, - created_at, expires_at + created_at, expires_at, + recall_count, recall_score, last_recalled_at FROM episodic_summaries WHERE agent_id = ? AND user_id = ? AND tenant_id = ? AND promoted_at IS NULL - ORDER BY created_at ASC LIMIT ?`, - agentID, userID, tenantID.String(), limit) + ORDER BY ` + orderBy + ` LIMIT ?` + rows, err := s.db.QueryContext(ctx, query, agentID, userID, tenantID.String(), limit) if err != nil { return nil, fmt.Errorf("episodic list_unpromoted: %w", err) } @@ -167,6 +182,35 @@ func (s *SQLiteEpisodicStore) ListUnpromoted(ctx context.Context, agentID, userI return scanSQLiteEpisodicRows(rows) } +// RecordRecall increments recall_count, folds `score` into the running +// average stored in recall_score, and sets last_recalled_at. Atomic single +// UPDATE; the SET clause reads OLD row values (SQLite evaluates expressions +// against the pre-update tuple), so the running average is computed +// correctly in one statement. +func (s *SQLiteEpisodicStore) RecordRecall(ctx context.Context, id string, score float64) error { + if id == "" { + return nil + } + if score < 0 { + score = 0 + } else if score > 1 { + score = 1 + } + tenantID := tenantIDForInsert(ctx) + now := time.Now().UTC().Format(time.RFC3339Nano) + _, err := s.db.ExecContext(ctx, ` + UPDATE episodic_summaries + SET recall_count = recall_count + 1, + recall_score = (recall_score * recall_count + ?) / (recall_count + 1), + last_recalled_at = ? + WHERE id = ? AND tenant_id = ?`, + score, now, id, tenantID.String()) + if err != nil { + return fmt.Errorf("episodic record_recall: %w", err) + } + return nil +} + // MarkPromoted sets promoted_at=now() for the given IDs. // IDs are processed in chunks of 200 to avoid SQLite variable limit. func (s *SQLiteEpisodicStore) MarkPromoted(ctx context.Context, ids []string) error { @@ -211,18 +255,22 @@ func (s *SQLiteEpisodicStore) CountUnpromoted(ctx context.Context, agentID, user return count, nil } -// scanSQLiteEpisodic scans a single row into EpisodicSummary. +// scanSQLiteEpisodic scans a single row into EpisodicSummary. Column order +// matches the SELECT lists in Get / List / ListUnpromoted* (17 columns incl. +// Phase 10 recall signals). func scanSQLiteEpisodic(row *sql.Row) (*store.EpisodicSummary, error) { var ep store.EpisodicSummary var idStr, tenantStr, agentStr string var topicsBytes []byte var createdAt sqliteTime var expiresAt nullSqliteTime + var lastRecalledAt nullSqliteTime err := row.Scan( &idStr, &tenantStr, &agentStr, &ep.UserID, &ep.SessionKey, &ep.Summary, &topicsBytes, &ep.TurnCount, &ep.TokenCount, &ep.L0Abstract, &ep.SourceID, &ep.SourceType, - &createdAt, &expiresAt) + &createdAt, &expiresAt, + &ep.RecallCount, &ep.RecallScore, &lastRecalledAt) if errors.Is(err, sql.ErrNoRows) { return nil, nil } @@ -238,6 +286,10 @@ func scanSQLiteEpisodic(row *sql.Row) (*store.EpisodicSummary, error) { t := expiresAt.Time ep.ExpiresAt = &t } + if lastRecalledAt.Valid { + t := lastRecalledAt.Time + ep.LastRecalledAt = &t + } return &ep, nil } @@ -250,11 +302,13 @@ func scanSQLiteEpisodicRows(rows *sql.Rows) ([]store.EpisodicSummary, error) { var topicsBytes []byte var createdAt sqliteTime var expiresAt nullSqliteTime + var lastRecalledAt nullSqliteTime if err := rows.Scan( &idStr, &tenantStr, &agentStr, &ep.UserID, &ep.SessionKey, &ep.Summary, &topicsBytes, &ep.TurnCount, &ep.TokenCount, &ep.L0Abstract, &ep.SourceID, &ep.SourceType, - &createdAt, &expiresAt); err != nil { + &createdAt, &expiresAt, + &ep.RecallCount, &ep.RecallScore, &lastRecalledAt); err != nil { return nil, err } ep.ID, _ = uuid.Parse(idStr) @@ -266,6 +320,10 @@ func scanSQLiteEpisodicRows(rows *sql.Rows) ([]store.EpisodicSummary, error) { t := expiresAt.Time ep.ExpiresAt = &t } + if lastRecalledAt.Valid { + t := lastRecalledAt.Time + ep.LastRecalledAt = &t + } results = append(results, ep) } return results, rows.Err() diff --git a/internal/store/sqlitestore/schema.go b/internal/store/sqlitestore/schema.go index c9a790d5..2b53e969 100644 --- a/internal/store/sqlitestore/schema.go +++ b/internal/store/sqlitestore/schema.go @@ -15,7 +15,7 @@ var schemaSQL string // SchemaVersion is the current SQLite schema version. // Bump this when adding new migration steps below. -const SchemaVersion = 12 +const SchemaVersion = 15 // migrations maps version → SQL to apply when upgrading FROM that version. // schema.sql always represents the LATEST full schema (for fresh DBs). @@ -366,6 +366,52 @@ WHERE a.deleted_at IS NULL AND NOT EXISTS (SELECT 1 FROM agent_context_files WHERE agent_id = a.id AND file_name = 'AGENTS_TASK.md'); DELETE FROM agent_context_files WHERE file_name = 'AGENTS_MINIMAL.md';`, + + // Version 12 → 13: Phase 10 dreaming weighted scoring signals on + // episodic_summaries. Mirrors PG migration 000045. + 12: `ALTER TABLE episodic_summaries ADD COLUMN recall_count INTEGER NOT NULL DEFAULT 0; +ALTER TABLE episodic_summaries ADD COLUMN recall_score REAL NOT NULL DEFAULT 0; +ALTER TABLE episodic_summaries ADD COLUMN last_recalled_at TEXT; +CREATE INDEX IF NOT EXISTS idx_episodic_recall_unpromoted ON episodic_summaries(agent_id, user_id, recall_score DESC) + WHERE promoted_at IS NULL;`, + + // Version 13 → 14: vault_documents agent_id nullable + unique index with tenant_id. + // SQLite requires table recreation to drop NOT NULL. Preserve all data. + 13: `CREATE TABLE vault_documents_new ( + id TEXT NOT NULL PRIMARY KEY, + tenant_id TEXT NOT NULL REFERENCES tenants(id) ON DELETE CASCADE, + agent_id TEXT REFERENCES agents(id) ON DELETE SET NULL, + team_id TEXT REFERENCES agent_teams(id) ON DELETE SET NULL, + scope TEXT NOT NULL DEFAULT 'personal', + custom_scope TEXT, + path TEXT NOT NULL, + title TEXT NOT NULL DEFAULT '', + doc_type TEXT NOT NULL DEFAULT 'note', + content_hash TEXT NOT NULL DEFAULT '', + summary TEXT NOT NULL DEFAULT '', + metadata TEXT DEFAULT '{}', + created_at TEXT DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now')), + updated_at TEXT DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now')) +); +INSERT INTO vault_documents_new SELECT * FROM vault_documents; +DROP TABLE vault_documents; +ALTER TABLE vault_documents_new RENAME TO vault_documents; +DROP INDEX IF EXISTS idx_vault_docs_unique_path; +CREATE UNIQUE INDEX idx_vault_docs_unique_path + ON vault_documents(tenant_id, COALESCE(agent_id, ''), COALESCE(team_id, ''), scope, path); +CREATE INDEX IF NOT EXISTS idx_vault_docs_tenant ON vault_documents(tenant_id); +CREATE INDEX IF NOT EXISTS idx_vault_docs_agent_scope ON vault_documents(agent_id, scope); +CREATE INDEX IF NOT EXISTS idx_vault_docs_type ON vault_documents(agent_id, doc_type); +CREATE INDEX IF NOT EXISTS idx_vault_docs_hash ON vault_documents(content_hash); +CREATE INDEX IF NOT EXISTS idx_vault_docs_team ON vault_documents(team_id);`, + + // Version 14 → 15: cron_jobs UNIQUE constraint on (agent_id, tenant_id, name). + // Dedup first: keep one row per combo (SQLite has no DISTINCT ON — use GROUP BY + MIN). + 14: `DELETE FROM cron_jobs WHERE id NOT IN ( + SELECT MIN(id) FROM cron_jobs GROUP BY agent_id, tenant_id, name +); +CREATE UNIQUE INDEX IF NOT EXISTS uq_cron_jobs_agent_tenant_name + ON cron_jobs(agent_id, tenant_id, name);`, } // EnsureSchema creates tables if they don't exist and applies incremental migrations. diff --git a/internal/store/sqlitestore/schema.sql b/internal/store/sqlitestore/schema.sql index eedf56e0..7efb3241 100644 --- a/internal/store/sqlitestore/schema.sql +++ b/internal/store/sqlitestore/schema.sql @@ -482,6 +482,7 @@ CREATE INDEX IF NOT EXISTS idx_cron_jobs_user_id ON cron_jobs(user_id); CREATE INDEX IF NOT EXISTS idx_cron_jobs_agent_user ON cron_jobs(agent_id, user_id); CREATE INDEX IF NOT EXISTS idx_cron_jobs_team ON cron_jobs(team_id) WHERE team_id IS NOT NULL; CREATE INDEX IF NOT EXISTS idx_cron_jobs_tenant ON cron_jobs(tenant_id); +CREATE UNIQUE INDEX IF NOT EXISTS uq_cron_jobs_agent_tenant_name ON cron_jobs(agent_id, tenant_id, name); -- ============================================================ -- Table: cron_run_logs @@ -1414,7 +1415,12 @@ CREATE TABLE IF NOT EXISTS episodic_summaries ( token_count INTEGER NOT NULL DEFAULT 0, created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now')), expires_at TEXT, - promoted_at TEXT + promoted_at TEXT, + -- Phase 10: dreaming weighted scoring signals (running avg of memory_search + -- hit scores). See internal/consolidation/scoring.go::ComputeRecallScore. + recall_count INTEGER NOT NULL DEFAULT 0, + recall_score REAL NOT NULL DEFAULT 0, + last_recalled_at TEXT ); CREATE INDEX IF NOT EXISTS idx_episodic_agent_user ON episodic_summaries(agent_id, user_id); @@ -1423,6 +1429,8 @@ CREATE UNIQUE INDEX IF NOT EXISTS idx_episodic_source_dedup ON episodic_summarie WHERE source_id IS NOT NULL; CREATE INDEX IF NOT EXISTS idx_episodic_unpromoted ON episodic_summaries(agent_id, user_id, created_at) WHERE promoted_at IS NULL; +CREATE INDEX IF NOT EXISTS idx_episodic_recall_unpromoted ON episodic_summaries(agent_id, user_id, recall_score DESC) + WHERE promoted_at IS NULL; -- ============================================================ -- Table: agent_evolution_metrics (V3 self-evolution Stage 1) @@ -1509,7 +1517,7 @@ CREATE INDEX IF NOT EXISTS idx_scuc_binary ON secure_cli_user_credentials(binary CREATE TABLE IF NOT EXISTS vault_documents ( id TEXT NOT NULL PRIMARY KEY, tenant_id TEXT NOT NULL REFERENCES tenants(id) ON DELETE CASCADE, - agent_id TEXT NOT NULL REFERENCES agents(id) ON DELETE CASCADE, + agent_id TEXT REFERENCES agents(id) ON DELETE SET NULL, team_id TEXT REFERENCES agent_teams(id) ON DELETE SET NULL, scope TEXT NOT NULL DEFAULT 'personal', custom_scope TEXT, @@ -1524,7 +1532,7 @@ CREATE TABLE IF NOT EXISTS vault_documents ( ); -- SQLite prohibits expressions in inline UNIQUE constraints; use a unique index instead. CREATE UNIQUE INDEX IF NOT EXISTS idx_vault_docs_unique_path - ON vault_documents(agent_id, COALESCE(team_id, ''), scope, path); + ON vault_documents(tenant_id, COALESCE(agent_id, ''), COALESCE(team_id, ''), scope, path); CREATE INDEX IF NOT EXISTS idx_vault_docs_tenant ON vault_documents(tenant_id); CREATE INDEX IF NOT EXISTS idx_vault_docs_agent_scope ON vault_documents(agent_id, scope); CREATE INDEX IF NOT EXISTS idx_vault_docs_type ON vault_documents(agent_id, doc_type); diff --git a/internal/store/sqlitestore/schema_migration_test.go b/internal/store/sqlitestore/schema_migration_test.go index 52855bbd..e3a9f6c9 100644 --- a/internal/store/sqlitestore/schema_migration_test.go +++ b/internal/store/sqlitestore/schema_migration_test.go @@ -50,20 +50,14 @@ func TestEnsureSchema_FreshDB(t *testing.T) { } } -// TestEnsureSchema_MigrationV11Only verifies the v11→12 patch (our new migration) -// applies correctly on a DB already at version 11. +// TestEnsureSchema_MigrationV11Only verifies migrations from v11 onward +// apply correctly on a DB built at version 11. func TestEnsureSchema_MigrationV11Only(t *testing.T) { - db := openTestDB(t) + db := openTestDBAtVersion(t, 11) - // Apply fresh schema first, then roll version back to 11 + // Re-apply — should run migrations 11→SchemaVersion if err := EnsureSchema(db); err != nil { - t.Fatalf("initial EnsureSchema: %v", err) - } - db.Exec("UPDATE schema_version SET version = 11") - - // Re-apply — should run only migration 11→12 - if err := EnsureSchema(db); err != nil { - t.Fatalf("EnsureSchema (v11→12) failed: %v", err) + t.Fatalf("EnsureSchema (v11→current) failed: %v", err) } var version int @@ -84,15 +78,12 @@ func TestEnsureSchema_IdempotentRerun(t *testing.T) { } } -// TestEnsureSchema_MigrationV11_SeedsAgentFiles verifies migration 11 seeds +// TestEnsureSchema_MigrationV11_SeedsAgentFiles verifies migration 11→12 seeds // AGENTS_CORE.md and AGENTS_TASK.md and removes AGENTS_MINIMAL.md. func TestEnsureSchema_MigrationV11_SeedsAgentFiles(t *testing.T) { - db := openTestDB(t) - if err := EnsureSchema(db); err != nil { - t.Fatalf("EnsureSchema: %v", err) - } + db := openTestDBAtVersion(t, 11) - // Use master tenant (seeded by EnsureSchema → seedMasterTenant) + // Use master tenant (seeded by seedMasterTenant) tenantID := "0193a5b0-7000-7000-8000-000000000001" agentID := "aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa" _, err := db.Exec(`INSERT INTO agents (id, tenant_id, agent_key, display_name, provider, model, agent_type, owner_id) @@ -107,10 +98,9 @@ func TestEnsureSchema_MigrationV11_SeedsAgentFiles(t *testing.T) { VALUES ('min-id', ?, 'AGENTS_MINIMAL.md', 'old minimal', ?, datetime('now'), datetime('now'))`, agentID, tenantID) - // Reset schema_version to 11 and re-apply migration - db.Exec("UPDATE schema_version SET version = 11") + // Re-apply — runs migrations 11→SchemaVersion (includes v11→12 seed) if err := EnsureSchema(db); err != nil { - t.Fatalf("EnsureSchema (re-apply v11): %v", err) + t.Fatalf("EnsureSchema (re-apply from v11): %v", err) } // Verify AGENTS_CORE.md seeded @@ -144,3 +134,49 @@ func openTestDB(t *testing.T) *sql.DB { t.Cleanup(func() { _ = db.Close() }) return db } + +// openTestDBAtVersion creates a fresh DB, applies full schema, then +// drops columns added by migrations > targetVersion so re-running +// EnsureSchema from that version exercises the real migration path. +// +// We accomplish this by applying schema at targetVersion: apply full +// schema.sql then set version = targetVersion. Migrations will ALTER +// TABLE ADD COLUMN — which only fails if the column already exists. +// To avoid that, we drop the columns that post-targetVersion migrations add. +func openTestDBAtVersion(t *testing.T, targetVersion int) *sql.DB { + t.Helper() + db := openTestDB(t) + + // Apply full schema first (creates all tables with all columns). + if err := EnsureSchema(db); err != nil { + t.Fatalf("EnsureSchema: %v", err) + } + + // Undo columns added by migrations after targetVersion. + // SQLite DROP COLUMN support varies, so recreate affected tables. + if targetVersion <= 11 { + // Migration 12 adds recall_count, recall_score, last_recalled_at. + // Recreate episodic_summaries without those columns. + db.Exec(`CREATE TABLE episodic_summaries_old AS SELECT + id, tenant_id, agent_id, user_id, session_key, summary, l0_abstract, + key_topics, source_type, source_id, turn_count, token_count, + created_at, expires_at, promoted_at + FROM episodic_summaries`) + db.Exec(`DROP TABLE episodic_summaries`) + db.Exec(`CREATE TABLE episodic_summaries ( + id TEXT NOT NULL PRIMARY KEY, tenant_id TEXT NOT NULL, agent_id TEXT NOT NULL, + user_id VARCHAR(255) NOT NULL DEFAULT '', session_key TEXT NOT NULL, + summary TEXT NOT NULL, l0_abstract TEXT NOT NULL DEFAULT '', + key_topics TEXT NOT NULL DEFAULT '[]', source_type TEXT NOT NULL DEFAULT 'session', + source_id TEXT, turn_count INTEGER NOT NULL DEFAULT 0, + token_count INTEGER NOT NULL DEFAULT 0, + created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')), + expires_at TEXT, promoted_at TEXT)`) + db.Exec(`INSERT INTO episodic_summaries SELECT * FROM episodic_summaries_old`) + db.Exec(`DROP TABLE episodic_summaries_old`) + } + + // Set version back to target. + db.Exec("UPDATE schema_version SET version = ?", targetVersion) + return db +} diff --git a/internal/store/sqlitestore/tenants.go b/internal/store/sqlitestore/tenants.go index f5bfba49..133f3b6d 100644 --- a/internal/store/sqlitestore/tenants.go +++ b/internal/store/sqlitestore/tenants.go @@ -7,6 +7,7 @@ import ( "database/sql" "encoding/json" "errors" + "strings" "time" "github.com/google/uuid" @@ -86,6 +87,33 @@ func (s *SQLiteTenantStore) ListTenants(ctx context.Context) ([]store.TenantData return result, nil } +func (s *SQLiteTenantStore) GetTenantsByIDs(ctx context.Context, ids []uuid.UUID) ([]store.TenantData, error) { + if len(ids) == 0 { + return nil, nil + } + const chunkSize = 500 + var all []store.TenantData + for start := 0; start < len(ids); start += chunkSize { + end := min(start+chunkSize, len(ids)) + chunk := ids[start:end] + ph := make([]string, len(chunk)) + args := make([]any, len(chunk)) + for i, id := range chunk { + ph[i] = "?" + args[i] = id.String() + } + q := `SELECT ` + tenantSelectCols + ` FROM tenants WHERE id IN (` + strings.Join(ph, ",") + `)` + var rows []tenantRow + if err := pkgSqlxDB.SelectContext(ctx, &rows, q, args...); err != nil { + return nil, err + } + for _, r := range rows { + all = append(all, r.toTenantData()) + } + } + return all, nil +} + func (s *SQLiteTenantStore) UpdateTenant(ctx context.Context, id uuid.UUID, updates map[string]any) error { return execMapUpdate(ctx, s.db, "tenants", id, updates) } diff --git a/internal/store/sqlitestore/vault_documents.go b/internal/store/sqlitestore/vault_documents.go index 5b8a5e7a..7852340d 100644 --- a/internal/store/sqlitestore/vault_documents.go +++ b/internal/store/sqlitestore/vault_documents.go @@ -58,11 +58,16 @@ func (s *SQLiteVaultStore) UpsertDocument(ctx context.Context, doc *store.VaultD meta = []byte("{}") } + // Convert nullable *string AgentID to nil for SQL. + var agentIDVal any + if doc.AgentID != nil && *doc.AgentID != "" { + agentIDVal = *doc.AgentID + } err = s.db.QueryRowContext(ctx, ` INSERT INTO vault_documents (id, tenant_id, agent_id, team_id, scope, custom_scope, path, title, doc_type, content_hash, summary, metadata, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) - ON CONFLICT (agent_id, COALESCE(team_id, '00000000-0000-0000-0000-000000000000'), scope, path) DO UPDATE SET + ON CONFLICT (tenant_id, COALESCE(agent_id,''), COALESCE(team_id,''), scope, path) DO UPDATE SET title = excluded.title, doc_type = excluded.doc_type, content_hash = excluded.content_hash, @@ -71,7 +76,7 @@ func (s *SQLiteVaultStore) UpsertDocument(ctx context.Context, doc *store.VaultD tenant_id = excluded.tenant_id, updated_at = excluded.updated_at RETURNING id`, - id, doc.TenantID, doc.AgentID, doc.TeamID, doc.Scope, doc.CustomScope, + id, doc.TenantID, agentIDVal, doc.TeamID, doc.Scope, doc.CustomScope, doc.Path, doc.Title, doc.DocType, doc.ContentHash, doc.Summary, string(meta), now, now, ).Scan(&doc.ID) if err != nil { @@ -81,11 +86,17 @@ func (s *SQLiteVaultStore) UpsertDocument(ctx context.Context, doc *store.VaultD } // GetDocument retrieves a vault document by tenant, agent, and path. +// Empty agentID means no agent filter. // Team scoping via RunContext: present+TeamID → filter; present+empty → personal; nil → any match. func (s *SQLiteVaultStore) GetDocument(ctx context.Context, tenantID, agentID, path string) (*store.VaultDocument, error) { q := `SELECT id, tenant_id, agent_id, team_id, scope, custom_scope, path, title, doc_type, content_hash, summary, metadata, created_at, updated_at - FROM vault_documents WHERE tenant_id = ? AND agent_id = ? AND path = ?` - args := []any{tenantID, agentID, path} + FROM vault_documents WHERE tenant_id = ? AND path = ?` + args := []any{tenantID, path} + + if agentID != "" { + q += " AND agent_id = ?" + args = append(args, agentID) + } if rc := store.RunContextFromCtx(ctx); rc != nil { if rc.TeamID != "" { @@ -108,11 +119,71 @@ func (s *SQLiteVaultStore) GetDocumentByID(ctx context.Context, tenantID, id str return scanVaultDoc(row) } +// GetDocumentsByIDs returns documents matching the given IDs with tenant isolation. +func (s *SQLiteVaultStore) GetDocumentsByIDs(ctx context.Context, tenantID string, docIDs []string) ([]store.VaultDocument, error) { + if len(docIDs) == 0 { + return nil, nil + } + const chunkSize = 500 + var all []store.VaultDocument + for start := 0; start < len(docIDs); start += chunkSize { + end := min(start+chunkSize, len(docIDs)) + chunk := docIDs[start:end] + ph := make([]string, len(chunk)) + args := make([]any, len(chunk)) + for i, id := range chunk { + ph[i] = "?" + args[i] = id + } + args = append(args, tenantID) + q := `SELECT id, tenant_id, agent_id, team_id, scope, custom_scope, path, title, doc_type, content_hash, summary, metadata, created_at, updated_at + FROM vault_documents WHERE id IN (` + strings.Join(ph, ",") + `) AND tenant_id = ?` + rows, err := s.db.QueryContext(ctx, q, args...) + if err != nil { + return nil, err + } + for rows.Next() { + doc, scanErr := scanVaultDocRow(rows) + if scanErr != nil { + rows.Close() + return nil, scanErr + } + all = append(all, *doc) + } + rows.Close() + if err := rows.Err(); err != nil { + return nil, err + } + } + return all, nil +} + +// GetDocumentByBasename finds a document by path basename (case-insensitive). +func (s *SQLiteVaultStore) GetDocumentByBasename(ctx context.Context, tenantID, agentID, basename string) (*store.VaultDocument, error) { + q := `SELECT id, tenant_id, agent_id, team_id, scope, custom_scope, path, title, doc_type, content_hash, summary, metadata, created_at, updated_at + FROM vault_documents + WHERE tenant_id = ? AND lower(replace(path, rtrim(path, replace(path, '/', '')), '')) = lower(?)` + args := []any{tenantID, basename} + if agentID != "" { + q += " AND agent_id = ?" + args = append(args, agentID) + } + q += " LIMIT 1" + row := s.db.QueryRowContext(ctx, q, args...) + return scanVaultDoc(row) +} + // DeleteDocument removes a vault document (FK cascades delete vault_links). +// Empty agentID means no agent filter. // Team scoping via RunContext (same rules as GetDocument). func (s *SQLiteVaultStore) DeleteDocument(ctx context.Context, tenantID, agentID, path string) error { - q := `DELETE FROM vault_documents WHERE tenant_id = ? AND agent_id = ? AND path = ?` - args := []any{tenantID, agentID, path} + q := `DELETE FROM vault_documents WHERE tenant_id = ? AND path = ?` + args := []any{tenantID, path} + + if agentID != "" { + q += " AND agent_id = ?" + args = append(args, agentID) + } if rc := store.RunContextFromCtx(ctx); rc != nil { if rc.TeamID != "" { @@ -249,9 +320,14 @@ func (s *SQLiteVaultStore) Search(ctx context.Context, opts store.VaultSearchOpt q := `SELECT id, tenant_id, agent_id, team_id, scope, custom_scope, path, title, doc_type, content_hash, summary, metadata, created_at, updated_at FROM vault_documents - WHERE tenant_id = ? AND agent_id = ? + WHERE tenant_id = ? AND (title LIKE ? ESCAPE '\' OR path LIKE ? ESCAPE '\')` - args := []any{opts.TenantID, opts.AgentID, pattern, pattern} + args := []any{opts.TenantID, pattern, pattern} + + if opts.AgentID != "" { + q += " AND agent_id = ?" + args = append(args, opts.AgentID) + } q, args = sqliteAppendTeamFilter(q, args, opts.TeamID, opts.TeamIDs) @@ -304,12 +380,14 @@ func (s *SQLiteVaultStore) Search(ctx context.Context, opts store.VaultSearchOpt func scanVaultDoc(row *sql.Row) (*store.VaultDocument, error) { var doc store.VaultDocument var meta []byte + var agentID *string ca, ua := &sqliteTime{}, &sqliteTime{} - err := row.Scan(&doc.ID, &doc.TenantID, &doc.AgentID, &doc.TeamID, &doc.Scope, &doc.CustomScope, + err := row.Scan(&doc.ID, &doc.TenantID, &agentID, &doc.TeamID, &doc.Scope, &doc.CustomScope, &doc.Path, &doc.Title, &doc.DocType, &doc.ContentHash, &doc.Summary, &meta, ca, ua) if err != nil { return nil, err } + doc.AgentID = agentID doc.CreatedAt = ca.Time doc.UpdatedAt = ua.Time if len(meta) > 2 { @@ -321,12 +399,14 @@ func scanVaultDoc(row *sql.Row) (*store.VaultDocument, error) { func scanVaultDocRow(rows *sql.Rows) (*store.VaultDocument, error) { var doc store.VaultDocument var meta []byte + var agentID *string ca, ua := &sqliteTime{}, &sqliteTime{} - err := rows.Scan(&doc.ID, &doc.TenantID, &doc.AgentID, &doc.TeamID, &doc.Scope, &doc.CustomScope, + err := rows.Scan(&doc.ID, &doc.TenantID, &agentID, &doc.TeamID, &doc.Scope, &doc.CustomScope, &doc.Path, &doc.Title, &doc.DocType, &doc.ContentHash, &doc.Summary, &meta, ca, ua) if err != nil { return nil, err } + doc.AgentID = agentID doc.CreatedAt = ca.Time doc.UpdatedAt = ua.Time if len(meta) > 2 { diff --git a/internal/store/sqlitestore/vault_links.go b/internal/store/sqlitestore/vault_links.go index c327aabe..ff78953c 100644 --- a/internal/store/sqlitestore/vault_links.go +++ b/internal/store/sqlitestore/vault_links.go @@ -6,12 +6,99 @@ import ( "context" "database/sql" "fmt" + "strings" "time" "github.com/google/uuid" "github.com/nextlevelbuilder/goclaw/internal/store" ) +// CreateLinks batch-inserts vault links with tenant validation. +func (s *SQLiteVaultStore) CreateLinks(ctx context.Context, links []store.VaultLink) error { + if len(links) == 0 { + return nil + } + + // Collect all unique doc IDs for tenant validation. + docIDSet := make(map[string]struct{}) + for _, l := range links { + docIDSet[l.FromDocID] = struct{}{} + docIDSet[l.ToDocID] = struct{}{} + } + docIDs := make([]string, 0, len(docIDSet)) + for id := range docIDSet { + docIDs = append(docIDs, id) + } + + // Batch-fetch tenant_id for all referenced docs. + type docMeta struct { + tenantID string + teamID *string + } + docMap := make(map[string]docMeta, len(docIDs)) + ph := make([]string, len(docIDs)) + args := make([]any, len(docIDs)) + for i, id := range docIDs { + ph[i] = "?" + args[i] = id + } + rows, err := s.db.QueryContext(ctx, + `SELECT id, tenant_id, team_id FROM vault_documents WHERE id IN (`+strings.Join(ph, ",")+`)`, args...) + if err != nil { + return fmt.Errorf("vault batch link: fetch docs: %w", err) + } + defer rows.Close() + for rows.Next() { + var id, tid string + var teamID *string + if err := rows.Scan(&id, &tid, &teamID); err != nil { + return fmt.Errorf("vault batch link: scan doc: %w", err) + } + docMap[id] = docMeta{tenantID: tid, teamID: teamID} + } + if err := rows.Err(); err != nil { + return fmt.Errorf("vault batch link: rows err: %w", err) + } + + // Validate and insert (chunk by 500). + const chunkSize = 500 + var valid []store.VaultLink + for _, l := range links { + from, fromOK := docMap[l.FromDocID] + to, toOK := docMap[l.ToDocID] + if !fromOK || !toOK { + continue + } + if from.tenantID != to.tenantID { + continue + } + if from.teamID != nil && to.teamID != nil && *from.teamID != *to.teamID { + continue + } + valid = append(valid, l) + } + + now := time.Now().UTC().Format(time.RFC3339Nano) + for start := 0; start < len(valid); start += chunkSize { + end := min(start+chunkSize, len(valid)) + chunk := valid[start:end] + const cols = 6 + iArgs := make([]any, 0, len(chunk)*cols) + iPh := make([]string, 0, len(chunk)) + for _, l := range chunk { + iPh = append(iPh, "(?,?,?,?,?,?)") + iArgs = append(iArgs, uuid.Must(uuid.NewV7()).String(), l.FromDocID, l.ToDocID, l.LinkType, l.Context, now) + } + q := `INSERT INTO vault_links (id, from_doc_id, to_doc_id, link_type, context, created_at) + VALUES ` + strings.Join(iPh, ",") + ` + ON CONFLICT (from_doc_id, to_doc_id, link_type) DO UPDATE SET context = excluded.context` + if _, err := s.db.ExecContext(ctx, q, iArgs...); err != nil { + return fmt.Errorf("vault batch create links: %w", err) + } + } + return nil +} + // CreateLink inserts a vault link, updating context on conflict. // Validates same-tenant + same-team boundary before insert. func (s *SQLiteVaultStore) CreateLink(ctx context.Context, link *store.VaultLink) error { @@ -77,6 +164,30 @@ func (s *SQLiteVaultStore) GetOutLinks(ctx context.Context, tenantID, docID stri return scanVaultLinkRows(rows) } +// GetOutLinksBatch returns all outlinks for multiple doc IDs in a single query. +func (s *SQLiteVaultStore) GetOutLinksBatch(ctx context.Context, tenantID string, docIDs []string) ([]store.VaultLink, error) { + if len(docIDs) == 0 { + return nil, nil + } + ph := strings.Repeat("?,", len(docIDs)-1) + "?" + args := make([]any, 0, len(docIDs)+1) + for _, id := range docIDs { + args = append(args, id) + } + args = append(args, tenantID) + rows, err := s.db.QueryContext(ctx, + `SELECT vl.id, vl.from_doc_id, vl.to_doc_id, vl.link_type, vl.context, vl.created_at + FROM vault_links vl + JOIN vault_documents d ON d.id = vl.from_doc_id + WHERE vl.from_doc_id IN (`+ph+`) AND d.tenant_id = ? + ORDER BY vl.created_at`, args...) + if err != nil { + return nil, err + } + defer rows.Close() + return scanVaultLinkRows(rows) +} + // GetBacklinks returns enriched backlinks pointing to a document (single JOIN, LIMIT 100). func (s *SQLiteVaultStore) GetBacklinks(ctx context.Context, tenantID, docID string) ([]store.VaultBacklink, error) { rows, err := s.db.QueryContext(ctx, ` @@ -111,6 +222,40 @@ func (s *SQLiteVaultStore) DeleteDocLinks(ctx context.Context, tenantID, docID s return err } +// DeleteDocLinksByType removes outbound links of a specific type from a document, scoped by tenant. +func (s *SQLiteVaultStore) DeleteDocLinksByType(ctx context.Context, tenantID, docID, linkType string) error { + _, err := s.db.ExecContext(ctx, ` + DELETE FROM vault_links + WHERE from_doc_id = ? + AND link_type = ? + AND from_doc_id IN (SELECT id FROM vault_documents WHERE tenant_id = ?)`, + docID, linkType, tenantID) + return err +} + +// DeleteDocLinksByTypes removes outbound links matching any of the given types from a document. +func (s *SQLiteVaultStore) DeleteDocLinksByTypes(ctx context.Context, tenantID, docID string, types []string) error { + if len(types) == 0 { + return nil + } + // Build IN clause with ? placeholders. + params := []any{docID} + placeholders := make([]string, len(types)) + for i, t := range types { + params = append(params, t) + placeholders[i] = "?" + } + params = append(params, tenantID) + q := fmt.Sprintf(` + DELETE FROM vault_links + WHERE from_doc_id = ? + AND link_type IN (%s) + AND from_doc_id IN (SELECT id FROM vault_documents WHERE tenant_id = ?)`, + strings.Join(placeholders, ",")) + _, err := s.db.ExecContext(ctx, q, params...) + return err +} + func scanVaultLinkRows(rows *sql.Rows) ([]store.VaultLink, error) { var links []store.VaultLink for rows.Next() { diff --git a/internal/store/tenant_store.go b/internal/store/tenant_store.go index 08d4b68f..e592fdac 100644 --- a/internal/store/tenant_store.go +++ b/internal/store/tenant_store.go @@ -67,6 +67,9 @@ type TenantStore interface { ListUsers(ctx context.Context, tenantID uuid.UUID) ([]TenantUserData, error) ListUserTenants(ctx context.Context, userID string) ([]TenantUserData, error) + // GetTenantsByIDs returns tenants matching the given UUIDs in a single query. + GetTenantsByIDs(ctx context.Context, ids []uuid.UUID) ([]TenantData, error) + // ResolveUserTenant returns the tenant_id for a user. // If user belongs to multiple tenants, returns the first (by created_at). // If no membership, returns MasterTenantID (backward compat). diff --git a/internal/store/vault_store.go b/internal/store/vault_store.go index 9fd570a1..7c09a9bc 100644 --- a/internal/store/vault_store.go +++ b/internal/store/vault_store.go @@ -9,7 +9,7 @@ import ( type VaultDocument struct { ID string `json:"id" db:"id"` TenantID string `json:"tenant_id" db:"tenant_id"` - AgentID string `json:"agent_id" db:"agent_id"` + AgentID *string `json:"agent_id,omitempty" db:"agent_id"` TeamID *string `json:"team_id,omitempty" db:"team_id"` Scope string `json:"scope" db:"scope"` // personal, team, shared CustomScope *string `json:"custom_scope,omitempty" db:"custom_scope"` @@ -83,15 +83,24 @@ type VaultStore interface { CountDocuments(ctx context.Context, tenantID, agentID string, opts VaultListOptions) (int, error) UpdateHash(ctx context.Context, tenantID, id, newHash string) error + // GetDocumentsByIDs returns documents matching the given IDs with tenant isolation. + GetDocumentsByIDs(ctx context.Context, tenantID string, docIDs []string) ([]VaultDocument, error) + // GetDocumentByBasename finds a document by path basename (case-insensitive). + GetDocumentByBasename(ctx context.Context, tenantID, agentID, basename string) (*VaultDocument, error) + // Search (FTS + vector hybrid) Search(ctx context.Context, opts VaultSearchOptions) ([]VaultSearchResult, error) // Links + CreateLinks(ctx context.Context, links []VaultLink) error CreateLink(ctx context.Context, link *VaultLink) error DeleteLink(ctx context.Context, tenantID, id string) error GetOutLinks(ctx context.Context, tenantID, docID string) ([]VaultLink, error) + GetOutLinksBatch(ctx context.Context, tenantID string, docIDs []string) ([]VaultLink, error) GetBacklinks(ctx context.Context, tenantID, docID string) ([]VaultBacklink, error) DeleteDocLinks(ctx context.Context, tenantID, docID string) error + DeleteDocLinksByType(ctx context.Context, tenantID, docID, linkType string) error + DeleteDocLinksByTypes(ctx context.Context, tenantID, docID string, types []string) error // Enrichment // UpdateSummaryAndReembed updates summary text and re-generates embedding from title+path+summary. diff --git a/internal/tools/memory.go b/internal/tools/memory.go index a14e62a7..2b8aefe1 100644 --- a/internal/tools/memory.go +++ b/internal/tools/memory.go @@ -180,10 +180,47 @@ func (t *MemorySearchTool) Execute(ctx context.Context, args map[string]any) *Re // Record retrieval metric non-blocking (best-effort). t.recordRetrievalMetric(ctx, len(combined), episodicResults) + // Phase 10: update per-episode recall signals for dreaming weighted + // scoring. Fire-and-forget — recall tracking must never block the hot + // search path or surface errors to the agent loop. + t.recordEpisodicRecall(ctx, episodicResults) return NewResult(string(data)) } +// recordEpisodicRecall schedules a best-effort RecordRecall update per +// episodic hit so DreamingWorker can prioritise useful entries. Runs in a +// background goroutine bounded by a 5s timeout so slow DBs can't leak. +func (t *MemorySearchTool) recordEpisodicRecall(ctx context.Context, episodic []store.EpisodicSearchResult) { + if t.episodicStore == nil || len(episodic) == 0 { + return + } + tenantID := store.TenantIDFromContext(ctx) + if tenantID == uuid.Nil { + return + } + // Snapshot hits so the goroutine doesn't observe caller mutations. + hits := make([]store.EpisodicSearchResult, 0, len(episodic)) + for _, r := range episodic { + if r.EpisodicID == "" { + continue + } + hits = append(hits, r) + } + if len(hits) == 0 { + return + } + go func() { + bgCtx, cancel := context.WithTimeout(store.WithTenantID(context.Background(), tenantID), 5*time.Second) + defer cancel() + for _, r := range hits { + if err := t.episodicStore.RecordRecall(bgCtx, r.EpisodicID, r.Score); err != nil { + slog.Debug("memory.recall.record_failed", "episodic_id", r.EpisodicID, "error", err) + } + } + }() +} + // recordRetrievalMetric records a memory_search retrieval metric in a background goroutine. func (t *MemorySearchTool) recordRetrievalMetric(ctx context.Context, resultCount int, episodic []store.EpisodicSearchResult) { if t.metricsStore == nil { diff --git a/internal/tools/vault_interceptor.go b/internal/tools/vault_interceptor.go index fc0746c2..e9d6597f 100644 --- a/internal/tools/vault_interceptor.go +++ b/internal/tools/vault_interceptor.go @@ -25,14 +25,15 @@ func NewVaultInterceptor(vs store.VaultStore, workspace string, bus eventbus.Dom return &VaultInterceptor{vaultStore: vs, workspace: workspace, eventBus: bus} } -// inferScopeFromContext returns scope and team_id based on RunContext. -// TeamID present → scope="team", teamID=&rc.TeamID. Absent → "personal", nil. -func inferScopeFromContext(ctx context.Context) (scope string, teamID *string) { +// inferScopeFromContext returns scope, team_id, and whether agent_id should be set. +// TeamID present → scope="team", teamID=&rc.TeamID, agentOwned=false. +// Absent → "personal", nil, agentOwned=true. +func inferScopeFromContext(ctx context.Context) (scope string, teamID *string, agentOwned bool) { rc := store.RunContextFromCtx(ctx) if rc != nil && rc.TeamID != "" { - return "team", &rc.TeamID + return "team", &rc.TeamID, false } - return "personal", nil + return "personal", nil, true } // AfterWrite registers or updates a vault document after a file write. @@ -56,13 +57,21 @@ func (v *VaultInterceptor) AfterWrite(ctx context.Context, resolvedPath, content } hash := vault.ContentHash([]byte(content)) - title := inferVaultTitle(relPath) - docType := inferVaultDocType(relPath) - scope, teamID := inferScopeFromContext(ctx) + title := vault.InferTitle(relPath) + docType := vault.InferDocType(relPath) + scope, teamID, agentOwned := inferScopeFromContext(ctx) + + // Team-scoped files belong to the team, not the creating agent. + var agentIDPtr *string + eventAgentID := "" + if agentOwned { + agentIDPtr = &agentID + eventAgentID = agentID + } doc := &store.VaultDocument{ TenantID: tenantID, - AgentID: agentID, + AgentID: agentIDPtr, TeamID: teamID, Scope: scope, Path: relPath, @@ -80,14 +89,14 @@ func (v *VaultInterceptor) AfterWrite(ctx context.Context, resolvedPath, content v.eventBus.Publish(eventbus.DomainEvent{ ID: uuid.Must(uuid.NewV7()).String(), Type: eventbus.EventVaultDocUpserted, - SourceID: doc.ID + ":" + hash, // unique per content version, avoids bus-level dedup suppression + SourceID: doc.ID + ":" + hash, TenantID: tenantID, - AgentID: agentID, + AgentID: eventAgentID, Timestamp: time.Now(), Payload: eventbus.VaultDocUpsertedPayload{ DocID: doc.ID, TenantID: tenantID, - AgentID: agentID, + AgentID: eventAgentID, Path: relPath, ContentHash: hash, Workspace: v.workspace, @@ -123,12 +132,19 @@ func (v *VaultInterceptor) AfterWriteMedia(ctx context.Context, resolvedPath, su return } - title := inferVaultTitle(relPath) - scope, teamID := inferScopeFromContext(ctx) + title := vault.InferTitle(relPath) + scope, teamID, agentOwned := inferScopeFromContext(ctx) + + var agentIDPtr *string + eventAgentID := "" + if agentOwned { + agentIDPtr = &agentID + eventAgentID = agentID + } doc := &store.VaultDocument{ TenantID: tenantID, - AgentID: agentID, + AgentID: agentIDPtr, TeamID: teamID, Scope: scope, Path: relPath, @@ -150,12 +166,12 @@ func (v *VaultInterceptor) AfterWriteMedia(ctx context.Context, resolvedPath, su Type: eventbus.EventVaultDocUpserted, SourceID: doc.ID + ":" + hash, TenantID: tenantID, - AgentID: agentID, + AgentID: eventAgentID, Timestamp: time.Now(), Payload: eventbus.VaultDocUpsertedPayload{ DocID: doc.ID, TenantID: tenantID, - AgentID: agentID, + AgentID: eventAgentID, Path: relPath, ContentHash: hash, Workspace: v.workspace, @@ -183,7 +199,11 @@ func (v *VaultInterceptor) BeforeRead(ctx context.Context, resolvedPath string) return } + // Try agent-scoped first, then tenant-wide (team/shared docs have no agent_id). doc, err := v.vaultStore.GetDocument(ctx, tenantID, agentID, relPath) + if err != nil { + doc, err = v.vaultStore.GetDocument(ctx, tenantID, "", relPath) + } if err != nil { return // not registered yet — skip } @@ -199,38 +219,3 @@ func (v *VaultInterceptor) BeforeRead(ctx context.Context, resolvedPath string) } } -// inferVaultTitle extracts a human-readable title from a file path. -func inferVaultTitle(relPath string) string { - base := filepath.Base(relPath) - ext := filepath.Ext(base) - return strings.TrimSuffix(base, ext) -} - -// inferVaultDocType guesses doc_type from path conventions. -// Media extension check runs first — doc_type describes content format, not location. -func inferVaultDocType(relPath string) string { - lower := strings.ToLower(relPath) - ext := strings.ToLower(filepath.Ext(relPath)) - - // Media types (images, video, audio) - switch ext { - case ".png", ".jpg", ".jpeg", ".gif", ".webp", ".svg", ".bmp", - ".mp4", ".webm", ".mov", ".avi", ".mkv", - ".mp3", ".wav", ".ogg", ".flac", ".aac", ".m4a": - return "media" - } - - // Path-based inference - switch { - case strings.HasPrefix(lower, "memory/"): - return "memory" - case strings.Contains(lower, "soul.md") || strings.Contains(lower, "identity.md") || strings.Contains(lower, "agents.md"): - return "context" - case strings.HasPrefix(lower, "skills/") || strings.HasSuffix(lower, "skill.md"): - return "skill" - case strings.HasPrefix(lower, "episodic/"): - return "episodic" - default: - return "note" - } -} diff --git a/internal/tools/vault_link.go b/internal/tools/vault_link.go deleted file mode 100644 index 19c5bdd4..00000000 --- a/internal/tools/vault_link.go +++ /dev/null @@ -1,233 +0,0 @@ -package tools - -import ( - "context" - "fmt" - "strings" - - "github.com/google/uuid" - - "github.com/nextlevelbuilder/goclaw/internal/store" -) - -// VaultLinkTool creates explicit links between vault documents. -type VaultLinkTool struct { - vaultStore store.VaultStore -} - -func NewVaultLinkTool() *VaultLinkTool { - return &VaultLinkTool{} -} - -func (t *VaultLinkTool) SetVaultStore(vs store.VaultStore) { - t.vaultStore = vs -} - -func (t *VaultLinkTool) Name() string { return "vault_link" } - -func (t *VaultLinkTool) Description() string { - return "Create an explicit link between two vault documents. Similar to [[wikilinks]] in markdown." -} - -func (t *VaultLinkTool) Parameters() map[string]any { - return map[string]any{ - "type": "object", - "properties": map[string]any{ - "from": map[string]any{ - "type": "string", - "description": "Source document path (workspace-relative)", - }, - "to": map[string]any{ - "type": "string", - "description": "Target document path (workspace-relative)", - }, - "context": map[string]any{ - "type": "string", - "description": "Optional note describing the link relationship", - }, - "link_type": map[string]any{ - "type": "string", - "description": "Link type: wikilink (default) or reference", - "enum": []string{"wikilink", "reference"}, - }, - }, - "required": []string{"from", "to"}, - } -} - -func (t *VaultLinkTool) Execute(ctx context.Context, args map[string]any) *Result { - fromPath, _ := args["from"].(string) - toPath, _ := args["to"].(string) - linkCtx, _ := args["context"].(string) - linkType, _ := args["link_type"].(string) - if linkType == "" { - linkType = "wikilink" - } - - if fromPath == "" || toPath == "" { - return ErrorResult("both 'from' and 'to' paths are required") - } - if linkType != "wikilink" && linkType != "reference" { - return ErrorResult("link_type must be 'wikilink' or 'reference'") - } - - agentID := store.AgentIDFromContext(ctx) - tenantID := store.TenantIDFromContext(ctx) - if t.vaultStore == nil || agentID == uuid.Nil { - return ErrorResult("vault not available") - } - - tid := tenantID.String() - aid := agentID.String() - - // Infer scope and team from context. - var teamID *string - scope := "personal" - if rc := store.RunContextFromCtx(ctx); rc != nil && rc.TeamID != "" { - teamID = &rc.TeamID - scope = "team" - } - - // Resolve source doc (auto-register if not in vault) - fromDoc, err := t.resolveOrRegister(ctx, tid, aid, teamID, scope, fromPath) - if err != nil { - return ErrorResult(fmt.Sprintf("cannot resolve source doc: %v", err)) - } - - // Resolve target doc (auto-register if not in vault) - toDoc, err := t.resolveOrRegister(ctx, tid, aid, teamID, scope, toPath) - if err != nil { - return ErrorResult(fmt.Sprintf("cannot resolve target doc: %v", err)) - } - - // Block cross-team links (both team docs must be same team). - if fromDoc.TeamID != nil && toDoc.TeamID != nil && *fromDoc.TeamID != *toDoc.TeamID { - return ErrorResult("cannot link documents from different teams") - } - - link := &store.VaultLink{ - FromDocID: fromDoc.ID, - ToDocID: toDoc.ID, - LinkType: linkType, - Context: linkCtx, - } - if err := t.vaultStore.CreateLink(ctx, link); err != nil { - return ErrorResult(fmt.Sprintf("failed to create link: %v", err)) - } - - return NewResult(fmt.Sprintf("Linked %s → %s", fromPath, toPath)) -} - -// resolveOrRegister finds a vault doc by path, or creates a stub entry with team context. -func (t *VaultLinkTool) resolveOrRegister(ctx context.Context, tenantID, agentID string, teamID *string, scope, path string) (*store.VaultDocument, error) { - doc, err := t.vaultStore.GetDocument(ctx, tenantID, agentID, path) - if err == nil && doc != nil { - return doc, nil - } - // Auto-register stub with team context. - doc = &store.VaultDocument{ - TenantID: tenantID, - AgentID: agentID, - TeamID: teamID, - Scope: scope, - Path: path, - Title: strings.TrimSuffix(path, ".md"), - DocType: inferVaultDocType(path), - } - if err := t.vaultStore.UpsertDocument(ctx, doc); err != nil { - return nil, err - } - return t.vaultStore.GetDocument(ctx, tenantID, agentID, path) -} - -// VaultBacklinksTool shows all documents linking to a specific document. -type VaultBacklinksTool struct { - vaultStore store.VaultStore -} - -func NewVaultBacklinksTool() *VaultBacklinksTool { - return &VaultBacklinksTool{} -} - -func (t *VaultBacklinksTool) SetVaultStore(vs store.VaultStore) { - t.vaultStore = vs -} - -func (t *VaultBacklinksTool) Name() string { return "vault_backlinks" } - -func (t *VaultBacklinksTool) Description() string { - return "Show all documents that link TO this document. Useful for tracing dependencies and references." -} - -func (t *VaultBacklinksTool) Parameters() map[string]any { - return map[string]any{ - "type": "object", - "properties": map[string]any{ - "path": map[string]any{ - "type": "string", - "description": "Document path to find backlinks for", - }, - }, - "required": []string{"path"}, - } -} - -func (t *VaultBacklinksTool) Execute(ctx context.Context, args map[string]any) *Result { - path, _ := args["path"].(string) - if path == "" { - return ErrorResult("path parameter is required") - } - - agentID := store.AgentIDFromContext(ctx) - tenantID := store.TenantIDFromContext(ctx) - if t.vaultStore == nil || agentID == uuid.Nil { - return ErrorResult("vault not available") - } - - doc, err := t.vaultStore.GetDocument(ctx, tenantID.String(), agentID.String(), path) - if err != nil { - return ErrorResult(fmt.Sprintf("document not found in vault: %s", path)) - } - - backlinks, err := t.vaultStore.GetBacklinks(ctx, tenantID.String(), doc.ID) - if err != nil { - return ErrorResult(fmt.Sprintf("failed to get backlinks: %v", err)) - } - - // Determine current team context for filtering. - var currentTeamID string - if rc := store.RunContextFromCtx(ctx); rc != nil { - currentTeamID = rc.TeamID - } - - var sb strings.Builder - sb.WriteString(fmt.Sprintf("Documents linking to %s:\n\n", path)) - count := 0 - for _, bl := range backlinks { - // Team boundary filter: - // - Team context: show same-team docs only (hide personal + other teams) - // - Personal context: show personal docs only (hide team docs) - if bl.TeamID != nil && *bl.TeamID != "" { - if currentTeamID == "" || *bl.TeamID != currentTeamID { - continue - } - } else { - // Personal doc — hide in team context (prevents exfiltration) - if currentTeamID != "" { - continue - } - } - - count++ - sb.WriteString(fmt.Sprintf("%d. %s (%s)", count, bl.Title, bl.Path)) - if bl.Context != "" { - sb.WriteString(fmt.Sprintf(" — \"%s\"", bl.Context)) - } - sb.WriteByte('\n') - } - - if count == 0 { - return NewResult(fmt.Sprintf("No documents link to %s", path)) - } - return NewResult(sb.String()) -} diff --git a/internal/tracing/cost.go b/internal/tracing/cost.go index aaa18a6d..85cd3ca4 100644 --- a/internal/tracing/cost.go +++ b/internal/tracing/cost.go @@ -7,12 +7,40 @@ import ( // CalculateCost computes the USD cost for a single LLM call based on token usage and pricing. // Returns 0 if pricing is nil. +// +// Semantics for reasoning/thinking tokens: +// +// All supported providers (OpenAI o3/o4-mini, Codex/GPT-5 Responses API, Anthropic +// extended thinking) report Usage.ThinkingTokens as a SUB-COUNT of Usage.CompletionTokens: +// - OpenAI: completion_tokens includes reasoning; completion_tokens_details.reasoning_tokens is the breakdown. +// - Codex: output_tokens includes reasoning; output_tokens_details.reasoning_tokens is the breakdown. +// - Anthropic: output_tokens is total output; we estimate thinking tokens from streamed thinking_delta chars. +// +// Thus CompletionTokens already bills thinking at the output rate by default. Only +// when an explicit ReasoningPerMillion rate is configured do we split the two and +// price the thinking portion separately — otherwise we'd double-count. func CalculateCost(pricing *config.ModelPricing, usage *providers.Usage) float64 { if pricing == nil || usage == nil { return 0 } cost := float64(usage.PromptTokens) * pricing.InputPerMillion / 1_000_000 - cost += float64(usage.CompletionTokens) * pricing.OutputPerMillion / 1_000_000 + + // Split completion tokens into visible output + thinking only when a distinct + // ReasoningPerMillion rate is set. Otherwise price the full CompletionTokens + // at OutputPerMillion — matches the provider billing semantics described above. + if pricing.ReasoningPerMillion > 0 && usage.ThinkingTokens > 0 { + visible := usage.CompletionTokens - usage.ThinkingTokens + if visible < 0 { + // Defensive: thinkingChars/4 estimate for Anthropic may exceed OutputTokens + // under unusual streaming conditions. Clamp to zero instead of going negative. + visible = 0 + } + cost += float64(visible) * pricing.OutputPerMillion / 1_000_000 + cost += float64(usage.ThinkingTokens) * pricing.ReasoningPerMillion / 1_000_000 + } else { + cost += float64(usage.CompletionTokens) * pricing.OutputPerMillion / 1_000_000 + } + if pricing.CacheReadPerMillion > 0 && usage.CacheReadTokens > 0 { cost += float64(usage.CacheReadTokens) * pricing.CacheReadPerMillion / 1_000_000 } diff --git a/internal/tracing/cost_test.go b/internal/tracing/cost_test.go new file mode 100644 index 00000000..f36c566c --- /dev/null +++ b/internal/tracing/cost_test.go @@ -0,0 +1,196 @@ +package tracing + +import ( + "math" + "testing" + + "github.com/nextlevelbuilder/goclaw/internal/config" + "github.com/nextlevelbuilder/goclaw/internal/providers" +) + +func floatEquals(a, b float64) bool { + return math.Abs(a-b) < 1e-9 +} + +func TestCalculateCost_NilInputs(t *testing.T) { + if got := CalculateCost(nil, nil); got != 0 { + t.Errorf("nil pricing + nil usage: got %v, want 0", got) + } + if got := CalculateCost(&config.ModelPricing{InputPerMillion: 1}, nil); got != 0 { + t.Errorf("nil usage: got %v, want 0", got) + } + if got := CalculateCost(nil, &providers.Usage{PromptTokens: 100}); got != 0 { + t.Errorf("nil pricing: got %v, want 0", got) + } +} + +func TestCalculateCost_PromptAndCompletion(t *testing.T) { + pricing := &config.ModelPricing{ + InputPerMillion: 3.0, // $3/M input tokens + OutputPerMillion: 15.0, // $15/M output tokens + } + usage := &providers.Usage{ + PromptTokens: 1_000_000, + CompletionTokens: 500_000, + } + // 1M * 3 + 0.5M * 15 = 3 + 7.5 = 10.5 + want := 10.5 + if got := CalculateCost(pricing, usage); !floatEquals(got, want) { + t.Errorf("got %v, want %v", got, want) + } +} + +func TestCalculateCost_CacheTokens(t *testing.T) { + pricing := &config.ModelPricing{ + InputPerMillion: 3.0, + OutputPerMillion: 15.0, + CacheReadPerMillion: 0.3, // 10% of input + CacheCreatePerMillion: 3.75, // 25% premium + } + usage := &providers.Usage{ + PromptTokens: 1_000_000, + CompletionTokens: 500_000, + CacheReadTokens: 2_000_000, + CacheCreationTokens: 100_000, + } + // 1M*3 + 0.5M*15 + 2M*0.3 + 0.1M*3.75 = 3 + 7.5 + 0.6 + 0.375 = 11.475 + want := 11.475 + if got := CalculateCost(pricing, usage); !floatEquals(got, want) { + t.Errorf("got %v, want %v", got, want) + } +} + +// TestCalculateCost_ThinkingTokens_SubCountNoDoubleCount verifies that when +// ReasoningPerMillion is NOT set, thinking tokens are already included in +// CompletionTokens and must not be double-charged. This is the critical +// regression gate for OpenAI/Codex/GPT-5 where CompletionTokens already +// includes reasoning as a sub-count. +func TestCalculateCost_ThinkingTokens_SubCountNoDoubleCount(t *testing.T) { + pricing := &config.ModelPricing{ + InputPerMillion: 3.0, + OutputPerMillion: 15.0, + // ReasoningPerMillion intentionally unset. + } + // Simulate an OpenAI o4-mini response: 1000 completion tokens of which + // 800 are reasoning. Provider billing = 1000 * output_rate (reasoning + // is inside the completion bucket already). + usage := &providers.Usage{ + PromptTokens: 100, + CompletionTokens: 1000, + ThinkingTokens: 800, + } + // 100*3 + 1000*15 = 0.0003 + 0.015 = 0.0153 + want := 100.0*3.0/1_000_000 + 1000.0*15.0/1_000_000 + if got := CalculateCost(pricing, usage); !floatEquals(got, want) { + t.Errorf("double-count regression: got %v, want %v", got, want) + } +} + +// TestCalculateCost_ThinkingTokens_WithDistinctReasoningRate verifies that +// when a distinct ReasoningPerMillion is configured, the completion bucket +// is split into visible output + thinking, and each priced independently. +// This supports pricing tiers where reasoning is cheaper or more expensive +// than the visible output (e.g. dev-economy pricing). +func TestCalculateCost_ThinkingTokens_WithDistinctReasoningRate(t *testing.T) { + pricing := &config.ModelPricing{ + InputPerMillion: 3.0, + OutputPerMillion: 15.0, + ReasoningPerMillion: 10.0, // thinking cheaper than visible output + } + usage := &providers.Usage{ + PromptTokens: 100_000, + CompletionTokens: 250_000, // 50k visible + 200k thinking + ThinkingTokens: 200_000, + } + // 100k*3 + 50k*15 + 200k*10 = 0.3 + 0.75 + 2.0 = 3.05 + want := 3.05 + if got := CalculateCost(pricing, usage); !floatEquals(got, want) { + t.Errorf("got %v, want %v", got, want) + } +} + +// TestCalculateCost_ThinkingTokens_AnthropicEstimateOverrun verifies the +// defensive clamp when Anthropic's thinkingChars/4 estimate exceeds the +// API's reported OutputTokens (which can happen under unusual streaming). +func TestCalculateCost_ThinkingTokens_AnthropicEstimateOverrun(t *testing.T) { + pricing := &config.ModelPricing{ + InputPerMillion: 3.0, + OutputPerMillion: 15.0, + ReasoningPerMillion: 20.0, + } + usage := &providers.Usage{ + PromptTokens: 100, + CompletionTokens: 100, // API says 100 output + ThinkingTokens: 150, // our estimate says 150 (overrun) + } + // visible clamped to 0, thinking priced at 20/M: + // 100*3 + 0*15 + 150*20 = 0.0003 + 0 + 0.003 = 0.0033 + want := 100.0*3.0/1_000_000 + 150.0*20.0/1_000_000 + if got := CalculateCost(pricing, usage); !floatEquals(got, want) { + t.Errorf("got %v, want %v", got, want) + } +} + +// TestCalculateCost_ZeroThinking_BackwardCompat verifies no behavior change +// for models without thinking tokens (non-reasoning models). +func TestCalculateCost_ZeroThinking_BackwardCompat(t *testing.T) { + pricing := &config.ModelPricing{ + InputPerMillion: 3.0, + OutputPerMillion: 15.0, + ReasoningPerMillion: 20.0, // set but no thinking tokens used + } + usage := &providers.Usage{ + PromptTokens: 1_000_000, + CompletionTokens: 500_000, + ThinkingTokens: 0, + } + // Same as TestCalculateCost_PromptAndCompletion: 10.5 + want := 10.5 + if got := CalculateCost(pricing, usage); !floatEquals(got, want) { + t.Errorf("got %v, want %v", got, want) + } +} + +// TestCalculateCost_OriginalBehaviorUnchanged ensures non-reasoning models +// with no ReasoningPerMillion configured get exactly the same cost as before +// Phase 1 (guards against regressions in the common path). +func TestCalculateCost_OriginalBehaviorUnchanged(t *testing.T) { + pricing := &config.ModelPricing{ + InputPerMillion: 3.0, + OutputPerMillion: 15.0, + } + // GPT-4o style call with no reasoning at all. + usage := &providers.Usage{ + PromptTokens: 1_000_000, + CompletionTokens: 500_000, + } + want := 10.5 // 3 + 7.5 + if got := CalculateCost(pricing, usage); !floatEquals(got, want) { + t.Errorf("original-path regression: got %v, want %v", got, want) + } +} + +func TestLookupPricing_ProviderQualified(t *testing.T) { + m := map[string]*config.ModelPricing{ + "anthropic/claude-opus-4": {InputPerMillion: 15.0, OutputPerMillion: 75.0}, + "claude-opus-4": {InputPerMillion: 10.0, OutputPerMillion: 50.0}, + } + // Provider-qualified takes precedence + p := LookupPricing(m, "anthropic", "claude-opus-4") + if p == nil || p.InputPerMillion != 15.0 { + t.Errorf("expected provider-qualified match, got %+v", p) + } + // Fallback to bare model name + p = LookupPricing(m, "unknown", "claude-opus-4") + if p == nil || p.InputPerMillion != 10.0 { + t.Errorf("expected bare model fallback, got %+v", p) + } + // Not found + if p := LookupPricing(m, "unknown", "unknown"); p != nil { + t.Errorf("expected nil, got %+v", p) + } + // Nil map + if p := LookupPricing(nil, "x", "y"); p != nil { + t.Errorf("expected nil for nil map, got %+v", p) + } +} diff --git a/internal/upgrade/version.go b/internal/upgrade/version.go index c51fe279..7ba0c007 100644 --- a/internal/upgrade/version.go +++ b/internal/upgrade/version.go @@ -2,4 +2,4 @@ package upgrade // RequiredSchemaVersion is the schema migration version this binary requires. // Bump this whenever adding a new SQL migration file. -const RequiredSchemaVersion uint = 44 +const RequiredSchemaVersion uint = 47 diff --git a/internal/vault/enrich_classify.go b/internal/vault/enrich_classify.go new file mode 100644 index 00000000..83252c15 --- /dev/null +++ b/internal/vault/enrich_classify.go @@ -0,0 +1,176 @@ +package vault + +import ( + "context" + "fmt" + "log/slog" + "maps" + "slices" + + "github.com/nextlevelbuilder/goclaw/internal/providers" + "github.com/nextlevelbuilder/goclaw/internal/store" +) + +const ( + classifyMaxTokens = 1024 + classifyTemperature = 0.1 + classifyCtxMaxLen = 256 // max context string length stored in DB + classifySummaryMaxChars = 300 // max summary chars in prompt (validated: 300 for accuracy) + classifyMaxSourceDocs = 20 // max source docs per classifyLinks call (validated: cap unbounded time) +) + +// validClassifyTypes — accepted link types stored directly in DB (aligned with UI vault-link-dialog.tsx). +var validClassifyTypes = map[string]bool{ + "reference": true, "depends_on": true, "extends": true, + "related": true, "supersedes": true, "contradicts": true, +} + +type classifyDoc struct { + DocID, Title, Path, Summary string +} + +type candidatePair struct { + Source, Candidate classifyDoc + Score float64 +} + +// classifyLinks orchestrates LLM-based link classification for enriched docs. +func (w *enrichWorker) classifyLinks(ctx context.Context, tenantID, agentID string, results []enriched) { + if w.provider == nil { + return + } + + capped := results + if len(capped) > classifyMaxSourceDocs { + capped = capped[:classifyMaxSourceDocs] + } + + // On SQLite (desktop/Lite), FindSimilarDocs returns nil (no pgvector) — classify is a no-op. + candidates := w.gatherCandidates(ctx, tenantID, agentID, capped) + if len(candidates) == 0 { + return + } + + allTypes := slices.Collect(maps.Keys(validClassifyTypes)) + allTypes = append(allTypes, "semantic") // clean up legacy links + + for sourceDocID, pairs := range candidates { + source := pairs[0].Source + candidateDocs := make([]classifyDoc, len(pairs)) + for i, p := range pairs { + candidateDocs[i] = p.Candidate + } + + system, user := buildClassifyPrompt(source, candidateDocs) + raw, err := w.callClassifyWithRetry(ctx, system, user) + if err != nil { + slog.Warn("vault.classify: llm_failed", "doc", sourceDocID, "err", err) + continue // SKIP fallback + } + + parsed, err := parseClassifyResponse(raw, len(candidateDocs)) + if err != nil { + hint := fmt.Sprintf("\n\nPrevious response was invalid JSON (error: %s). Output ONLY a valid JSON array.", err.Error()) + raw2, err2 := w.callClassifyWithRetry(ctx, system, user+hint) + if err2 != nil { + slog.Warn("vault.classify: retry_parse_failed", "doc", sourceDocID, "err", err2) + continue + } + parsed, err = parseClassifyResponse(raw2, len(candidateDocs)) + if err != nil { + slog.Warn("vault.classify: parse_still_failed", "doc", sourceDocID, "err", err) + continue + } + } + + // Collect valid links (collect-then-write pattern). + var newLinks []store.VaultLink + for _, r := range parsed { + if r.Type == "SKIP" || !validClassifyTypes[r.Type] { + continue + } + linkCtx := r.Ctx + if len(linkCtx) > classifyCtxMaxLen { + linkCtx = string([]rune(linkCtx)[:classifyCtxMaxLen]) + } + newLinks = append(newLinks, store.VaultLink{ + FromDocID: sourceDocID, + ToDocID: candidateDocs[r.Idx-1].DocID, // idx is 1-based, validated by parseClassifyResponse + LinkType: r.Type, + Context: linkCtx, + }) + } + + // Only replace old links if LLM produced valid replacements (avoid data loss on all-SKIP). + if len(newLinks) > 0 { + if err := w.vault.DeleteDocLinksByTypes(ctx, tenantID, sourceDocID, allTypes); err != nil { + slog.Warn("vault.classify: delete_old", "doc", sourceDocID, "err", err) + } + if err := w.vault.CreateLinks(ctx, newLinks); err != nil { + slog.Debug("vault.classify: batch_create_links", "from", sourceDocID, "count", len(newLinks), "err", err) + } + } + } +} + +func (w *enrichWorker) gatherCandidates(ctx context.Context, tenantID, agentID string, results []enriched) map[string][]candidatePair { + seen := make(map[string]bool) + out := make(map[string][]candidatePair) + + for _, r := range results { + neighbors, err := w.vault.FindSimilarDocs(ctx, tenantID, agentID, r.payload.DocID, enrichSimilarityLimit) + if err != nil { + slog.Warn("vault.classify: find_similar", "doc", r.payload.DocID, "err", err) + continue + } + // Use title carried from Phase 0 batch-fetch (avoids per-doc refetch). + title := r.title + if title == "" { + title = r.payload.Path + } + src := classifyDoc{ + DocID: r.payload.DocID, + Title: title, + Path: r.payload.Path, + Summary: truncateSummary(r.summary), + } + for _, n := range neighbors { + if n.Score < enrichSimilarityMin || n.Document.Summary == "" { + continue + } + // Bidirectional dedup: only process each pair once. + a, b := src.DocID, n.Document.ID + if a > b { + a, b = b, a + } + if key := a + ":" + b; seen[key] { + continue + } else { + seen[key] = true + } + out[src.DocID] = append(out[src.DocID], candidatePair{ + Source: src, + Candidate: classifyDoc{ + DocID: n.Document.ID, + Title: n.Document.Title, + Path: n.Document.Path, + Summary: truncateSummary(n.Document.Summary), + }, + Score: n.Score, + }) + } + } + return out +} + +// callClassifyWithRetry calls the LLM with shared retry logic. +func (w *enrichWorker) callClassifyWithRetry(ctx context.Context, system, user string) (string, error) { + return w.chatWithRetry(ctx, "vault.classify", providers.ChatRequest{ + Messages: []providers.Message{ + {Role: "system", Content: system}, + {Role: "user", Content: user}, + }, + Model: w.model, + Options: map[string]any{"max_tokens": classifyMaxTokens, "temperature": classifyTemperature}, + }) +} diff --git a/internal/vault/enrich_classify_parse_edge_test.go b/internal/vault/enrich_classify_parse_edge_test.go new file mode 100644 index 00000000..3059100d --- /dev/null +++ b/internal/vault/enrich_classify_parse_edge_test.go @@ -0,0 +1,186 @@ +package vault + +import ( + "testing" +) + +// ============================================================================ +// JSON Parsing Edge Case Tests +// ============================================================================ + +// TestParseClassifyResponse_MixedValidInvalid handles mix of valid and invalid entries. +func TestParseClassifyResponse_MixedValidInvalid(t *testing.T) { + raw := `[ + {"idx":1,"type":"reference","ctx":"valid"}, + {"idx":0,"type":"reference","ctx":"invalid idx"}, + {"idx":2,"type":"bad_type","ctx":"invalid type"}, + {"idx":3,"type":"extends","ctx":"valid"} + ]` + + results, err := parseClassifyResponse(raw, 3) + if err != nil { + t.Fatalf("parseClassifyResponse failed: %v", err) + } + + // Should keep only idx 1 and 3 with valid types + if len(results) != 2 { + t.Fatalf("Expected 2 valid results, got %d", len(results)) + } + + if results[0].Idx != 1 || results[1].Idx != 3 { + t.Errorf("Invalid entries not filtered correctly") + } +} + +// TestParseClassifyResponse_NegativeIdx filters silently. +func TestParseClassifyResponse_NegativeIdx(t *testing.T) { + raw := `[ + {"idx":-1,"type":"reference","ctx":"invalid"}, + {"idx":1,"type":"reference","ctx":"valid"} + ]` + + results, err := parseClassifyResponse(raw, 1) + if err != nil { + t.Fatalf("parseClassifyResponse failed: %v", err) + } + + if len(results) != 1 { + t.Fatalf("Expected 1 result (negative idx filtered), got %d", len(results)) + } + + if results[0].Idx != 1 { + t.Errorf("Negative idx should be filtered") + } +} + +// TestParseClassifyResponse_AllSKIP preserves all SKIP entries. +func TestParseClassifyResponse_AllSKIP(t *testing.T) { + raw := `[ + {"idx":1,"type":"SKIP","ctx":"no match"}, + {"idx":2,"type":"SKIP","ctx":"different topic"} + ]` + + results, err := parseClassifyResponse(raw, 2) + if err != nil { + t.Fatalf("parseClassifyResponse failed: %v", err) + } + + if len(results) != 2 { + t.Fatalf("Expected 2 SKIP entries, got %d", len(results)) + } + + for i, r := range results { + if r.Type != "SKIP" { + t.Errorf("Result %d: expected SKIP, got %q", i, r.Type) + } + } +} + +// TestParseClassifyResponse_LargeIdx beyond count are filtered. +func TestParseClassifyResponse_LargeIdx(t *testing.T) { + raw := `[ + {"idx":1,"type":"reference","ctx":"ok"}, + {"idx":100,"type":"reference","ctx":"too large"}, + {"idx":999,"type":"reference","ctx":"way too large"} + ]` + + results, err := parseClassifyResponse(raw, 5) + if err != nil { + t.Fatalf("parseClassifyResponse failed: %v", err) + } + + if len(results) != 1 { + t.Fatalf("Expected 1 result (oversized idx filtered), got %d", len(results)) + } + + if results[0].Idx != 1 { + t.Errorf("Large idx should be filtered") + } +} + +// TestParseClassifyResponse_MaxBoundaryIdx at exactly count is valid. +func TestParseClassifyResponse_MaxBoundaryIdx(t *testing.T) { + raw := `[ + {"idx":1,"type":"reference","ctx":"first"}, + {"idx":5,"type":"reference","ctx":"last valid"} + ]` + + results, err := parseClassifyResponse(raw, 5) + if err != nil { + t.Fatalf("parseClassifyResponse failed: %v", err) + } + + if len(results) != 2 { + t.Fatalf("Expected 2 results, got %d", len(results)) + } + + if results[1].Idx != 5 { + t.Errorf("Max boundary idx (5) should be valid when count=5") + } +} + +// TestParseClassifyResponse_EmptyCtx preserves empty context string. +func TestParseClassifyResponse_EmptyCtx(t *testing.T) { + raw := `[ + {"idx":1,"type":"reference","ctx":""}, + {"idx":2,"type":"related","ctx":"has context"} + ]` + + results, err := parseClassifyResponse(raw, 2) + if err != nil { + t.Fatalf("parseClassifyResponse failed: %v", err) + } + + if len(results) != 2 { + t.Fatalf("Expected 2 results, got %d", len(results)) + } + + if results[0].Ctx != "" { + t.Errorf("Empty ctx should be preserved, got %q", results[0].Ctx) + } +} + +// TestParseClassifyResponse_WhitespaceCtx preserves whitespace-only context. +func TestParseClassifyResponse_WhitespaceCtx(t *testing.T) { + raw := `[ + {"idx":1,"type":"reference","ctx":" "}, + {"idx":2,"type":"related","ctx":"valid"} + ]` + + results, err := parseClassifyResponse(raw, 2) + if err != nil { + t.Fatalf("parseClassifyResponse failed: %v", err) + } + + if len(results) != 2 { + t.Fatalf("Expected 2 results, got %d", len(results)) + } + + // Whitespace should be preserved + if results[0].Ctx != " " { + t.Errorf("Whitespace ctx should be preserved, got %q", results[0].Ctx) + } +} + +// TestParseClassifyResponse_CaseInsensitiveType only SKIP and validClassifyTypes are accepted. +func TestParseClassifyResponse_CaseInsensitiveType(t *testing.T) { + // Lowercase versions of valid types should not be accepted + raw := `[ + {"idx":1,"type":"reference","ctx":"ok"}, + {"idx":2,"type":"REFERENCE","ctx":"uppercase"} + ]` + + results, err := parseClassifyResponse(raw, 2) + if err != nil { + t.Fatalf("parseClassifyResponse failed: %v", err) + } + + // Only lowercase "reference" should be valid + if len(results) != 1 { + t.Fatalf("Expected 1 result (uppercase REFERENCE filtered), got %d", len(results)) + } + + if results[0].Type != "reference" { + t.Errorf("Only lowercase types accepted, got %q", results[0].Type) + } +} diff --git a/internal/vault/enrich_classify_parse_test.go b/internal/vault/enrich_classify_parse_test.go new file mode 100644 index 00000000..3400103e --- /dev/null +++ b/internal/vault/enrich_classify_parse_test.go @@ -0,0 +1,193 @@ +package vault + +import ( + "fmt" + "strings" + "testing" +) + +// ============================================================================ +// JSON Parsing Tests +// ============================================================================ + +// TestParseClassifyResponse_Valid parses standard JSON array. +func TestParseClassifyResponse_Valid(t *testing.T) { + raw := `[ + {"idx":1,"type":"reference","ctx":"mentions config"}, + {"idx":2,"type":"depends_on","ctx":"requires authentication"} + ]` + + results, err := parseClassifyResponse(raw, 2) + if err != nil { + t.Fatalf("parseClassifyResponse failed: %v", err) + } + + if len(results) != 2 { + t.Fatalf("Expected 2 results, got %d", len(results)) + } + + if results[0].Idx != 1 || results[0].Type != "reference" { + t.Errorf("Result 0: got idx=%d type=%q, want idx=1 type=reference", results[0].Idx, results[0].Type) + } + + if results[1].Idx != 2 || results[1].Type != "depends_on" { + t.Errorf("Result 1: got idx=%d type=%q, want idx=2 type=depends_on", results[1].Idx, results[1].Type) + } +} + +// TestParseClassifyResponse_WithCodeFence strips ```json fences. +func TestParseClassifyResponse_WithCodeFence(t *testing.T) { + raw := "```json\n[{\"idx\":1,\"type\":\"reference\",\"ctx\":\"test\"}]\n```" + + results, err := parseClassifyResponse(raw, 1) + if err != nil { + t.Fatalf("parseClassifyResponse with code fence failed: %v", err) + } + + if len(results) != 1 { + t.Fatalf("Expected 1 result, got %d", len(results)) + } + + if results[0].Type != "reference" { + t.Errorf("Expected type reference, got %q", results[0].Type) + } +} + +// TestParseClassifyResponse_InvalidJSON returns error on unmarshal failure. +func TestParseClassifyResponse_InvalidJSON(t *testing.T) { + raw := `{invalid json}` + + _, err := parseClassifyResponse(raw, 1) + if err == nil { + t.Fatalf("parseClassifyResponse should return error for invalid JSON") + } + + if !strings.Contains(err.Error(), "unmarshal") { + t.Errorf("Error should mention unmarshal: %v", err) + } +} + +// TestParseClassifyResponse_OutOfRangeIdx filters silently, keeps valid entries. +func TestParseClassifyResponse_OutOfRangeIdx(t *testing.T) { + raw := `[ + {"idx":1,"type":"reference","ctx":"ok"}, + {"idx":99,"type":"related","ctx":"invalid idx"}, + {"idx":2,"type":"extends","ctx":"ok"} + ]` + + results, err := parseClassifyResponse(raw, 2) + if err != nil { + t.Fatalf("parseClassifyResponse failed: %v", err) + } + + if len(results) != 2 { + t.Fatalf("Expected 2 results (out-of-range filtered), got %d", len(results)) + } + + if results[0].Idx != 1 || results[1].Idx != 2 { + t.Errorf("Out-of-range entry not filtered properly") + } +} + +// TestParseClassifyResponse_UnknownType filters silently. +func TestParseClassifyResponse_UnknownType(t *testing.T) { + raw := `[ + {"idx":1,"type":"reference","ctx":"ok"}, + {"idx":2,"type":"unknown_type","ctx":"invalid"}, + {"idx":3,"type":"related","ctx":"ok"} + ]` + + results, err := parseClassifyResponse(raw, 3) + if err != nil { + t.Fatalf("parseClassifyResponse failed: %v", err) + } + + if len(results) != 2 { + t.Fatalf("Expected 2 results (unknown type filtered), got %d", len(results)) + } + + if results[0].Type != "reference" || results[1].Type != "related" { + t.Errorf("Unknown type not filtered properly") + } +} + +// TestParseClassifyResponse_SKIPType preserves SKIP in output. +func TestParseClassifyResponse_SKIPType(t *testing.T) { + raw := `[ + {"idx":1,"type":"reference","ctx":"valid"}, + {"idx":2,"type":"SKIP","ctx":"no relationship"} + ]` + + results, err := parseClassifyResponse(raw, 2) + if err != nil { + t.Fatalf("parseClassifyResponse failed: %v", err) + } + + if len(results) != 2 { + t.Fatalf("Expected 2 results including SKIP, got %d", len(results)) + } + + skipFound := false + for _, r := range results { + if r.Type == "SKIP" { + skipFound = true + break + } + } + + if !skipFound { + t.Errorf("SKIP type should be preserved in results") + } +} + +// TestParseClassifyResponse_EmptyArray returns valid empty slice. +func TestParseClassifyResponse_EmptyArray(t *testing.T) { + raw := `[]` + + results, err := parseClassifyResponse(raw, 5) + if err != nil { + t.Fatalf("parseClassifyResponse failed on empty array: %v", err) + } + + if len(results) != 0 { + t.Fatalf("Expected 0 results, got %d", len(results)) + } +} + +// TestParseClassifyResponse_CtxTruncation truncates context over 256 chars. +func TestParseClassifyResponse_CtxTruncation(t *testing.T) { + longCtx := strings.Repeat("x", 300) + raw := fmt.Sprintf(`[{"idx":1,"type":"reference","ctx":"%s"}]`, longCtx) + + results, err := parseClassifyResponse(raw, 1) + if err != nil { + t.Fatalf("parseClassifyResponse failed: %v", err) + } + + if len([]rune(results[0].Ctx)) > classifyCtxMaxLen { + t.Errorf("Context not truncated: got %d runes, want ≤%d", + len([]rune(results[0].Ctx)), classifyCtxMaxLen) + } +} + +// TestParseClassifyResponse_ZeroIdx filters silently (idx must be >= 1). +func TestParseClassifyResponse_ZeroIdx(t *testing.T) { + raw := `[ + {"idx":0,"type":"reference","ctx":"invalid"}, + {"idx":1,"type":"reference","ctx":"valid"} + ]` + + results, err := parseClassifyResponse(raw, 1) + if err != nil { + t.Fatalf("parseClassifyResponse failed: %v", err) + } + + if len(results) != 1 { + t.Fatalf("Expected 1 result (idx 0 filtered), got %d", len(results)) + } + + if results[0].Idx != 1 { + t.Errorf("Zero idx should be filtered") + } +} + diff --git a/internal/vault/enrich_classify_prompt.go b/internal/vault/enrich_classify_prompt.go new file mode 100644 index 00000000..a02654c8 --- /dev/null +++ b/internal/vault/enrich_classify_prompt.go @@ -0,0 +1,90 @@ +package vault + +import ( + "encoding/json" + "fmt" + "log/slog" + "strings" +) + +// classifyResult is a single LLM classification output for a candidate doc. +type classifyResult struct { + Idx int `json:"idx"` + Type string `json:"type"` + Ctx string `json:"ctx"` +} + +const classifySystemPrompt = `You classify relationships between documents in a knowledge vault. + +## Link Types +- reference: A cites, mentions, or quotes B +- depends_on: A requires B to function (config, data dependency, prerequisite) +- extends: A expands, details, or elaborates on B +- related: A and B share the same topic or domain (use only when no stronger type fits) +- supersedes: A is a newer version that replaces or updates B +- contradicts: A conflicts with or opposes B's content + +## Rules +- Output ONLY a valid JSON array, no other text +- Use SKIP when documents are similar but have no meaningful relationship +- Prefer specific types (reference, depends_on) over generic (related) +- Each candidate classified independently +- Keep ctx descriptions under 60 words + +## Output Format +[{"idx":1,"type":"reference","ctx":"mentions OAuth config for setup"},{"idx":2,"type":"SKIP"}]` + +// buildClassifyPrompt formats the system and user prompts for classify LLM call. +func buildClassifyPrompt(source classifyDoc, candidates []classifyDoc) (system, user string) { + var b strings.Builder + b.WriteString("## Source Document\n") + fmt.Fprintf(&b, "Title: %s\nPath: %s\nSummary: %s\n\n", source.Title, source.Path, source.Summary) + b.WriteString("## Candidates\n") + for i, c := range candidates { + fmt.Fprintf(&b, "%d. Title: %s | Path: %s | Summary: %s\n", i+1, c.Title, c.Path, c.Summary) + } + return classifySystemPrompt, b.String() +} + +// parseClassifyResponse parses LLM JSON output into classify results. +// Uses partial success model: invalid entries filtered silently, error only on total unmarshal failure. +func parseClassifyResponse(raw string, count int) ([]classifyResult, error) { + raw = strings.TrimSpace(raw) + // Strip code fences. + raw = strings.TrimPrefix(raw, "```json") + raw = strings.TrimPrefix(raw, "```") + raw = strings.TrimSuffix(raw, "```") + raw = strings.TrimSpace(raw) + + var results []classifyResult + if err := json.Unmarshal([]byte(raw), &results); err != nil { + return nil, fmt.Errorf("json unmarshal: %w", err) + } + + // Filter invalid entries (partial success). + valid := results[:0] + for _, r := range results { + if r.Idx < 1 || r.Idx > count { + slog.Debug("vault.classify: idx out of range", "idx", r.Idx, "count", count) + continue + } + if r.Type != "SKIP" && !validClassifyTypes[r.Type] { + continue + } + // Truncate context. + if len(r.Ctx) > classifyCtxMaxLen { + r.Ctx = string([]rune(r.Ctx)[:classifyCtxMaxLen]) + } + valid = append(valid, r) + } + return valid, nil +} + +// truncateSummary caps summary at classifySummaryMaxChars with UTF-8 safe rune truncation. +func truncateSummary(s string) string { + runes := []rune(s) + if len(runes) <= classifySummaryMaxChars { + return s + } + return string(runes[:classifySummaryMaxChars]) + "..." +} diff --git a/internal/vault/enrich_classify_prompt_test.go b/internal/vault/enrich_classify_prompt_test.go new file mode 100644 index 00000000..776d5aaa --- /dev/null +++ b/internal/vault/enrich_classify_prompt_test.go @@ -0,0 +1,148 @@ +package vault + +import ( + "strings" + "testing" +) + +// ============================================================================ +// Prompt Building Tests +// ============================================================================ + +// TestBuildClassifyPrompt verifies system prompt contains all 6 types and user prompt has numbered candidates. +func TestBuildClassifyPrompt(t *testing.T) { + source := classifyDoc{ + DocID: "doc1", + Title: "Test Document", + Path: "docs/test.md", + Summary: "This is a test document.", + } + + candidates := []classifyDoc{ + { + DocID: "doc2", + Title: "Related Doc 1", + Path: "docs/related1.md", + Summary: "First related document.", + }, + { + DocID: "doc3", + Title: "Related Doc 2", + Path: "docs/related2.md", + Summary: "Second related document.", + }, + } + + system, user := buildClassifyPrompt(source, candidates) + + // System prompt must contain all 6 relationship types + expectedTypes := []string{"reference", "depends_on", "extends", "related", "supersedes", "contradicts"} + for _, typ := range expectedTypes { + if !strings.Contains(system, typ) { + t.Errorf("System prompt missing type: %s", typ) + } + } + + // System prompt must contain classifySystemPrompt constant text + if !strings.Contains(system, "Link Types") { + t.Errorf("System prompt missing 'Link Types' header") + } + + // User prompt must contain source doc info + if !strings.Contains(user, "Test Document") { + t.Errorf("User prompt missing source title") + } + if !strings.Contains(user, "docs/test.md") { + t.Errorf("User prompt missing source path") + } + if !strings.Contains(user, "This is a test document") { + t.Errorf("User prompt missing source summary") + } + + // User prompt must contain candidates with numbers (1., 2.) + if !strings.Contains(user, "1.") { + t.Errorf("User prompt missing numbered candidate (1.)") + } + if !strings.Contains(user, "2.") { + t.Errorf("User prompt missing numbered candidate (2.)") + } + + // User prompt must contain candidate info + if !strings.Contains(user, "Related Doc 1") { + t.Errorf("User prompt missing candidate 1 title") + } + if !strings.Contains(user, "Related Doc 2") { + t.Errorf("User prompt missing candidate 2 title") + } +} + +// ============================================================================ +// Truncate Summary Tests +// ============================================================================ + +// TestTruncateSummary_Long truncates at classifySummaryMaxChars (300) with "..." suffix. +func TestTruncateSummary_Long(t *testing.T) { + // Create string longer than 300 chars + longText := strings.Repeat("a", 350) + result := truncateSummary(longText) + + if len([]rune(result)) > classifySummaryMaxChars+3 { // +3 for "..." + t.Errorf("truncateSummary(%d chars) returned %d runes, want ≤%d", + len([]rune(longText)), len([]rune(result)), classifySummaryMaxChars+3) + } + + if !strings.HasSuffix(result, "...") { + t.Errorf("Truncated result must end with '...', got: %q", result) + } + + // Should be exactly 303 runes (300 + "...") + expected := len([]rune(strings.Repeat("a", classifySummaryMaxChars))) + 3 + if len([]rune(result)) != expected { + t.Errorf("Truncated result has %d runes, want %d", len([]rune(result)), expected) + } +} + +// TestTruncateSummary_Short returns short string as-is. +func TestTruncateSummary_Short(t *testing.T) { + short := "This is short" + result := truncateSummary(short) + + if result != short { + t.Errorf("truncateSummary(%q) = %q, want unchanged", short, result) + } + + if strings.Contains(result, "...") { + t.Errorf("Short string should not have '...' suffix: %q", result) + } +} + +// TestTruncateSummary_BoundaryAt300 tests string exactly at 300 chars. +func TestTruncateSummary_BoundaryAt300(t *testing.T) { + exact := strings.Repeat("x", classifySummaryMaxChars) + result := truncateSummary(exact) + + if result != exact { + t.Errorf("String at exactly %d chars should not be truncated", classifySummaryMaxChars) + } + + if strings.HasSuffix(result, "...") { + t.Errorf("String at boundary should not have '...' suffix") + } +} + +// TestTruncateSummary_Unicode tests UTF-8 safe truncation with multi-byte chars. +func TestTruncateSummary_Unicode(t *testing.T) { + // Use emoji (4 bytes each) to test multi-byte handling + longUnicode := strings.Repeat("🎉", 400) // 400 emoji chars, very long when encoded + result := truncateSummary(longUnicode) + + runes := []rune(result) + if len(runes) > classifySummaryMaxChars+3 { + t.Errorf("Unicode truncation exceeded max: got %d runes, want ≤%d", + len(runes), classifySummaryMaxChars+3) + } + + if !strings.HasSuffix(result, "...") { + t.Errorf("Truncated unicode must end with '...'") + } +} diff --git a/internal/vault/enrich_classify_retry_advanced_test.go b/internal/vault/enrich_classify_retry_advanced_test.go new file mode 100644 index 00000000..44613058 --- /dev/null +++ b/internal/vault/enrich_classify_retry_advanced_test.go @@ -0,0 +1,154 @@ +package vault + +import ( + "context" + "errors" + "strings" + "testing" +) + +// ============================================================================ +// Advanced Retry Logic Tests +// ============================================================================ + +// TestCallClassifyWithRetry_FirstAttemptSuccess uses first timeout. +func TestCallClassifyWithRetry_FirstAttemptSuccess(t *testing.T) { + // Verify that first attempt uses enrichRetryTimeouts[0] + if enrichRetryTimeouts[0] == 0 { + t.Errorf("First timeout should be non-zero") + } + + provider := &mockClassifyProvider{ + responses: []string{`[{"idx":1,"type":"reference","ctx":"first attempt"}]`}, + errors: []error{nil}, + } + + worker := &enrichWorker{ + provider: provider, + model: "test", + } + + ctx := context.Background() + resp, err := worker.callClassifyWithRetry(ctx, "system", "user") + + if err != nil { + t.Fatalf("First attempt should succeed: %v", err) + } + + if provider.calls != 1 { + t.Errorf("Expected exactly 1 call on first-attempt success, got %d", provider.calls) + } + + if !strings.Contains(resp, "reference") { + t.Errorf("Response missing expected content: %q", resp) + } +} + +// TestCallClassifyWithRetry_ResponseWhitespaceStripping trims whitespace from response. +func TestCallClassifyWithRetry_ResponseWhitespaceStripping(t *testing.T) { + // Response has leading/trailing whitespace that should be stripped + provider := &mockClassifyProvider{ + responses: []string{"\n\n [{\"idx\":1,\"type\":\"reference\",\"ctx\":\"test\"}] \n"}, + errors: []error{nil}, + } + + worker := &enrichWorker{ + provider: provider, + model: "test", + } + + ctx := context.Background() + resp, err := worker.callClassifyWithRetry(ctx, "system", "user") + + if err != nil { + t.Fatalf("callClassifyWithRetry failed: %v", err) + } + + // Response should be trimmed + if strings.HasPrefix(resp, "\n") || strings.HasSuffix(resp, "\n") { + t.Errorf("Response should be trimmed: %q", resp) + } + + if !strings.Contains(resp, "reference") { + t.Errorf("Response missing expected content: %q", resp) + } +} + +// TestCallClassifyWithRetry_MaxRetriesConstant verifies retry limit. +func TestCallClassifyWithRetry_MaxRetriesConstant(t *testing.T) { + if enrichMaxRetries != 3 { + t.Errorf("enrichMaxRetries should be 3, got %d", enrichMaxRetries) + } + + // Verify arrays have correct length + if len(enrichRetryTimeouts) != enrichMaxRetries { + t.Errorf("enrichRetryTimeouts length should be %d, got %d", enrichMaxRetries, len(enrichRetryTimeouts)) + } + + if len(enrichRetryBackoffs) != enrichMaxRetries { + t.Errorf("enrichRetryBackoffs length should be %d, got %d", enrichMaxRetries, len(enrichRetryBackoffs)) + } +} + +// TestCallClassifyWithRetry_SecondAttemptSucceeds verifies first retry succeeds. +func TestCallClassifyWithRetry_SecondAttemptSucceeds(t *testing.T) { + provider := &mockClassifyProvider{ + responses: []string{ + "", // attempt 0: error + `[{"idx":1,"type":"extends","ctx":"second attempt"}]`, // attempt 1: success + }, + errors: []error{ + errors.New("first attempt failed"), + nil, // no error on second attempt + }, + } + + worker := &enrichWorker{ + provider: provider, + model: "test", + } + + ctx := context.Background() + resp, err := worker.callClassifyWithRetry(ctx, "system", "user") + + if err != nil { + t.Fatalf("Should succeed on second attempt, got error: %v", err) + } + + if provider.calls != 2 { + t.Errorf("Expected 2 calls, got %d", provider.calls) + } + + if !strings.Contains(resp, "extends") { + t.Errorf("Response from second attempt not found: %q", resp) + } +} + +// TestCallClassifyWithRetry_EmptyResponse returns error for empty response on all attempts. +func TestCallClassifyWithRetry_EmptyResponse(t *testing.T) { + provider := &mockClassifyProvider{ + responses: []string{"", "", ""}, + errors: []error{nil, nil, nil}, + } + + worker := &enrichWorker{ + provider: provider, + model: "test", + } + + ctx := context.Background() + resp, err := worker.callClassifyWithRetry(ctx, "system", "user") + + // Empty response is still a successful LLM call, should return empty string + if err != nil { + t.Fatalf("Empty response should not error: %v", err) + } + + if resp != "" { + t.Errorf("Expected empty response, got %q", resp) + } + + if provider.calls != 1 { + t.Errorf("Expected 1 call (succeeded immediately), got %d", provider.calls) + } +} diff --git a/internal/vault/enrich_classify_retry_test.go b/internal/vault/enrich_classify_retry_test.go new file mode 100644 index 00000000..22cf3294 --- /dev/null +++ b/internal/vault/enrich_classify_retry_test.go @@ -0,0 +1,195 @@ +package vault + +import ( + "context" + "fmt" + "strings" + "testing" + + "github.com/nextlevelbuilder/goclaw/internal/providers" +) + +// ============================================================================ +// Mock Provider for Testing +// ============================================================================ + +// mockClassifyProvider implements providers.Provider for testing retry logic. +type mockClassifyProvider struct { + responses []string + errors []error + calls int +} + +func (m *mockClassifyProvider) Chat(ctx context.Context, req providers.ChatRequest) (*providers.ChatResponse, error) { + idx := m.calls + m.calls++ + if idx < len(m.errors) && m.errors[idx] != nil { + return nil, m.errors[idx] + } + resp := "" + if idx < len(m.responses) { + resp = m.responses[idx] + } + return &providers.ChatResponse{Content: resp}, nil +} + +func (m *mockClassifyProvider) ChatStream(ctx context.Context, req providers.ChatRequest, onChunk func(providers.StreamChunk)) (*providers.ChatResponse, error) { + return m.Chat(ctx, req) +} + +func (m *mockClassifyProvider) DefaultModel() string { return "test" } + +func (m *mockClassifyProvider) Name() string { return "mock" } + +// ============================================================================ +// Retry Logic Tests +// ============================================================================ + +// TestCallClassifyWithRetry_Success succeeds on first attempt. +func TestCallClassifyWithRetry_Success(t *testing.T) { + provider := &mockClassifyProvider{ + responses: []string{`[{"idx":1,"type":"reference","ctx":"test"}]`}, + errors: []error{nil}, + } + + worker := &enrichWorker{ + provider: provider, + model: "test", + } + + ctx := context.Background() + resp, err := worker.callClassifyWithRetry(ctx, "system", "user") + + if err != nil { + t.Fatalf("callClassifyWithRetry failed: %v", err) + } + + if !strings.Contains(resp, "reference") { + t.Errorf("Response doesn't contain expected content: %q", resp) + } + + if provider.calls != 1 { + t.Errorf("Expected 1 call, got %d", provider.calls) + } +} + +// TestCallClassifyWithRetry_RetryThenSuccess fails twice, succeeds on third attempt. +func TestCallClassifyWithRetry_RetryThenSuccess(t *testing.T) { + provider := &mockClassifyProvider{ + responses: []string{ + "", // attempt 0: error + "", // attempt 1: error + `[{"idx":1,"type":"reference","ctx":"success"}]`, // attempt 2: success + }, + errors: []error{ + fmt.Errorf("network error"), + fmt.Errorf("timeout"), + nil, // no error on third attempt + }, + } + + worker := &enrichWorker{ + provider: provider, + model: "test", + } + + ctx := context.Background() + resp, err := worker.callClassifyWithRetry(ctx, "system", "user") + + if err != nil { + t.Fatalf("callClassifyWithRetry should succeed after retries, got error: %v", err) + } + + if !strings.Contains(resp, "success") { + t.Errorf("Response doesn't contain expected content: %q", resp) + } + + if provider.calls != 3 { + t.Errorf("Expected 3 calls, got %d", provider.calls) + } +} + +// TestCallClassifyWithRetry_AllFail exhausts retries and returns error. +func TestCallClassifyWithRetry_AllFail(t *testing.T) { + provider := &mockClassifyProvider{ + responses: []string{"", "", ""}, + errors: []error{ + fmt.Errorf("error1"), + fmt.Errorf("error2"), + fmt.Errorf("error3"), + }, + } + + worker := &enrichWorker{ + provider: provider, + model: "test", + } + + ctx := context.Background() + _, err := worker.callClassifyWithRetry(ctx, "system", "user") + + if err == nil { + t.Fatalf("callClassifyWithRetry should return error after exhausting retries") + } + + if !strings.Contains(err.Error(), "exhausted") { + t.Errorf("Error should mention exhausted retries: %v", err) + } + + if provider.calls != enrichMaxRetries { + t.Errorf("Expected %d calls, got %d", enrichMaxRetries, provider.calls) + } +} + +// TestCallClassifyWithRetry_ContextCancellation respects context cancellation. +func TestCallClassifyWithRetry_ContextCancellation(t *testing.T) { + provider := &mockClassifyProvider{ + responses: []string{"", "", ""}, + errors: []error{ + fmt.Errorf("error1"), + fmt.Errorf("error2"), + fmt.Errorf("error3"), + }, + } + + worker := &enrichWorker{ + provider: provider, + model: "test", + } + + ctx, cancel := context.WithCancel(context.Background()) + cancel() // Cancel immediately + + _, err := worker.callClassifyWithRetry(ctx, "system", "user") + + if err == nil { + t.Fatalf("callClassifyWithRetry should return error for cancelled context") + } + + if !strings.Contains(err.Error(), "context") { + t.Errorf("Error should mention context: %v", err) + } +} + +// TestCallClassifyWithRetry_RetriesAndBackoffs verifies retry timeouts and backoffs escalate. +func TestCallClassifyWithRetry_RetriesAndBackoffs(t *testing.T) { + // Verify first backoff is 0 (no backoff on first attempt) + if enrichRetryBackoffs[0] != 0 { + t.Errorf("First backoff should be 0, got %v", enrichRetryBackoffs[0]) + } + + // Verify backoffs escalate for retry attempts + if enrichRetryBackoffs[1] == 0 || enrichRetryBackoffs[2] == 0 { + t.Errorf("Backoffs should escalate: %v", enrichRetryBackoffs) + } + + if enrichRetryBackoffs[1] >= enrichRetryBackoffs[2] { + t.Errorf("Backoffs should increase: %v", enrichRetryBackoffs) + } + + // Verify timeouts escalate + if enrichRetryTimeouts[0] >= enrichRetryTimeouts[1] || enrichRetryTimeouts[1] >= enrichRetryTimeouts[2] { + t.Errorf("Timeouts should escalate: %v", enrichRetryTimeouts) + } +} + diff --git a/internal/vault/enrich_progress.go b/internal/vault/enrich_progress.go new file mode 100644 index 00000000..89b55736 --- /dev/null +++ b/internal/vault/enrich_progress.go @@ -0,0 +1,78 @@ +package vault + +import ( + "sync" + + "github.com/google/uuid" + "github.com/nextlevelbuilder/goclaw/internal/bus" + "github.com/nextlevelbuilder/goclaw/pkg/protocol" +) + +// EnrichProgress tracks enrichment pipeline progress and broadcasts via WS events. +type EnrichProgress struct { + mu sync.Mutex + msgBus bus.EventPublisher + tenantID uuid.UUID + total int + done int + running bool +} + +// NewEnrichProgress creates a progress tracker that broadcasts to WS clients. +func NewEnrichProgress(msgBus bus.EventPublisher) *EnrichProgress { + return &EnrichProgress{msgBus: msgBus} +} + +// EnrichEvent is the WS event payload for vault enrichment progress. +type EnrichEvent struct { + Phase string `json:"phase"` // enriching, complete + Done int `json:"done"` // docs completed so far + Total int `json:"total"` // total docs in pipeline + Running bool `json:"running"` // false when pipeline idle +} + +// Status returns current progress state (for polling fallback / HTTP endpoint). +func (p *EnrichProgress) Status() EnrichEvent { + p.mu.Lock() + defer p.mu.Unlock() + phase := "enriching" + if !p.running { + phase = "idle" + } + return EnrichEvent{Phase: phase, Done: p.done, Total: p.total, Running: p.running} +} + +func (p *EnrichProgress) broadcast(e EnrichEvent) { + if p.msgBus == nil { + return + } + bus.BroadcastForTenant(p.msgBus, protocol.EventVaultEnrichProgress, p.tenantID, e) +} + +// Start signals enrichment with updated total and tenant scope. +func (p *EnrichProgress) Start(total int, tenantID uuid.UUID) { + p.mu.Lock() + defer p.mu.Unlock() + p.total = total + p.tenantID = tenantID + p.running = true + p.broadcast(EnrichEvent{Phase: "enriching", Done: p.done, Total: p.total, Running: true}) +} + +// AddDone increments completed count by n and broadcasts progress. +func (p *EnrichProgress) AddDone(n int) { + p.mu.Lock() + defer p.mu.Unlock() + p.done += n + p.broadcast(EnrichEvent{Phase: "enriching", Done: p.done, Total: p.total, Running: true}) +} + +// Finish signals the enrichment pipeline has completed and resets counters. +func (p *EnrichProgress) Finish() { + p.mu.Lock() + defer p.mu.Unlock() + p.broadcast(EnrichEvent{Phase: "complete", Done: p.done, Total: p.total, Running: false}) + p.done = 0 + p.total = 0 + p.running = false +} diff --git a/internal/vault/enrich_worker.go b/internal/vault/enrich_worker.go index 06c998e7..fe2c36a3 100644 --- a/internal/vault/enrich_worker.go +++ b/internal/vault/enrich_worker.go @@ -2,8 +2,12 @@ package vault import ( "context" + "encoding/json" "fmt" "log/slog" + + "github.com/google/uuid" + "github.com/nextlevelbuilder/goclaw/internal/bus" "os" "path/filepath" "strings" @@ -13,14 +17,23 @@ import ( "github.com/nextlevelbuilder/goclaw/internal/eventbus" "github.com/nextlevelbuilder/goclaw/internal/providers" "github.com/nextlevelbuilder/goclaw/internal/store" + "golang.org/x/sync/semaphore" ) const ( enrichMaxDedupEntries = 10000 - enrichContentMaxRunes = 8192 - enrichLLMTimeout = 5 * time.Minute - enrichSimilarityLimit = 5 + enrichSimilarityLimit = 10 enrichSimilarityMin = 0.7 + enrichMaxConcurrent = 3 // max concurrent batch summarize calls across chunks + enrichBatchSize = 5 // docs per enrichment chunk (1 LLM call per chunk) + enrichBatchItemMaxRunes = 3000 // per-file content limit in batch summarize + enrichMaxRetries = 3 // shared retry count for LLM calls (summarize + classify) +) + +// Shared retry config for all enrichment LLM calls. +var ( + enrichRetryTimeouts = [enrichMaxRetries]time.Duration{5 * time.Minute, 7 * time.Minute, 10 * time.Minute} + enrichRetryBackoffs = [enrichMaxRetries]time.Duration{0, 2 * time.Second, 4 * time.Second} ) // EnrichWorkerDeps bundles dependencies for the vault enrichment worker. @@ -29,18 +42,23 @@ type EnrichWorkerDeps struct { Provider providers.Provider Model string EventBus eventbus.DomainEventBus + MsgBus bus.EventPublisher // for WS event broadcast } // RegisterEnrichWorker subscribes the enrichment worker to vault doc events. -// Returns an unsubscribe function for cleanup. -func RegisterEnrichWorker(deps EnrichWorkerDeps) func() { +// Returns (unsubscribe func, progress tracker for WS broadcast). +func RegisterEnrichWorker(deps EnrichWorkerDeps) (func(), *EnrichProgress) { + progress := NewEnrichProgress(deps.MsgBus) w := &enrichWorker{ vault: deps.VaultStore, provider: deps.Provider, model: deps.Model, dedup: make(map[string]string), + sem: semaphore.NewWeighted(enrichMaxConcurrent), + progress: progress, } - return deps.EventBus.Subscribe(eventbus.EventVaultDocUpserted, w.Handle) + unsub := deps.EventBus.Subscribe(eventbus.EventVaultDocUpserted, w.Handle) + return unsub, progress } // enrichWorker processes vault document upsert events to generate summaries, @@ -50,10 +68,12 @@ type enrichWorker struct { provider providers.Provider model string queue enrichBatchQueue + progress *EnrichProgress // Bounded dedup: docID → content_hash. Prevents re-processing unchanged files. dedupMu sync.Mutex dedup map[string]string + sem *semaphore.Weighted // limits concurrent LLM summarize calls } // Handle is the EventBus handler for vault.doc_upserted events. @@ -71,7 +91,14 @@ func (w *enrichWorker) Handle(ctx context.Context, event eventbus.DomainEvent) e } w.dedupMu.Unlock() - key := payload.TenantID + ":" + payload.AgentID + // Batch key: tenant + agent for agent-scoped docs. + // Team/shared docs (empty AgentID) batch together per tenant so bulk rescan + // benefits from chunked processing instead of 1-doc-per-queue. + batchScope := payload.AgentID + if batchScope == "" { + batchScope = "_shared" + } + key := payload.TenantID + ":" + batchScope if !w.queue.Enqueue(key, payload) { return nil // another goroutine already processing this agent's queue } @@ -80,138 +107,302 @@ func (w *enrichWorker) Handle(ctx context.Context, event eventbus.DomainEvent) e return nil } +// enriched holds a successfully summarized vault document pending embed+link. +type enriched struct { + payload eventbus.VaultDocUpsertedPayload + summary string + title string // carried from DB for classify phase (avoids refetch) +} + // processBatch drains and processes queued vault doc events in a loop. +// Items are chunked into enrichBatchSize groups so bulk rescan doesn't +// overwhelm the LLM provider with hundreds of concurrent requests. func (w *enrichWorker) processBatch(ctx context.Context, key string) { + var totalQueued int + for { items := w.queue.Drain(key) if len(items) == 0 { if w.queue.TryFinish(key) { + if totalQueued > 0 { + w.progress.Finish() + } return } continue } - type enriched struct { - payload eventbus.VaultDocUpsertedPayload - summary string + totalQueued += len(items) + tenantID := uuid.Nil + if len(items) > 0 { + tenantID, _ = uuid.Parse(items[0].TenantID) } - var results []enriched + w.progress.Start(totalQueued, tenantID) - for _, item := range items { - // Dedup check. - w.dedupMu.Lock() - if prev, exists := w.dedup[item.DocID]; exists && prev == item.ContentHash { - w.dedupMu.Unlock() - continue + // Process in chunks of enrichBatchSize, up to enrichMaxConcurrent in parallel. + var wg sync.WaitGroup + for start := 0; start < len(items); start += enrichBatchSize { + end := min(start+enrichBatchSize, len(items)) + if err := w.sem.Acquire(ctx, 1); err != nil { + break } - w.dedupMu.Unlock() - - // Check if doc already has a summary (e.g., media with caption). - existing, err := w.vault.GetDocumentByID(ctx, item.TenantID, item.DocID) - if err != nil { - slog.Warn("vault.enrich: get_doc", "doc", item.DocID, "err", err) - continue - } - if existing != nil && existing.Summary != "" { - // Already has summary — skip LLM, still embed+link. - results = append(results, enriched{payload: item, summary: existing.Summary}) - continue - } - - // Read file content from disk. - fullPath := filepath.Join(item.Workspace, item.Path) - content, err := os.ReadFile(fullPath) - if err != nil { - slog.Warn("vault.enrich: read_file", "path", item.Path, "err", err) - continue // file deleted or moved — don't record in dedup - } - - // UTF-8 safe truncation. - runes := []rune(string(content)) - if len(runes) > enrichContentMaxRunes { - runes = runes[:enrichContentMaxRunes] - } - text := string(runes) - - // LLM summarize. - sctx, cancel := context.WithTimeout(ctx, enrichLLMTimeout) - summary, err := w.summarize(sctx, item.Path, text) - cancel() - if err != nil { - slog.Warn("vault.enrich: summarize", "path", item.Path, "err", err) - continue // don't record in dedup — allow retry on next write - } - - results = append(results, enriched{payload: item, summary: summary}) - } - - // Update summary + embed + auto-link for each enriched doc. - for _, r := range results { - if err := w.vault.UpdateSummaryAndReembed(ctx, r.payload.TenantID, r.payload.DocID, r.summary); err != nil { - slog.Warn("vault.enrich: update_summary", "doc", r.payload.DocID, "err", err) - continue // don't record in dedup - } - - // Record hash only after successful update. - w.recordDedup(r.payload.DocID, r.payload.ContentHash) - - // Auto-link via vector similarity. - w.autoLink(ctx, r.payload.TenantID, r.payload.AgentID, r.payload.DocID) + wg.Add(1) + go func(chunk []eventbus.VaultDocUpsertedPayload) { + defer wg.Done() + defer w.sem.Release(1) + w.processChunk(ctx, chunk) + w.progress.AddDone(len(chunk)) + }(items[start:end]) } + wg.Wait() if w.queue.TryFinish(key) { + w.progress.Finish() return } } } -const vaultSummarizePrompt = `Summarize this document in 2-3 sentences. Focus on: -- Main topic and purpose -- Key concepts, entities, or decisions -- Actionable information - -Be concise. Output only the summary, no preamble.` - -// summarize calls LLM to generate a short summary of the document. -func (w *enrichWorker) summarize(ctx context.Context, path, content string) (string, error) { - resp, err := w.provider.Chat(ctx, providers.ChatRequest{ - Messages: []providers.Message{ - {Role: "system", Content: vaultSummarizePrompt}, - {Role: "user", Content: "File: " + path + "\n\n" + content}, - }, - Model: w.model, - Options: map[string]any{"max_tokens": 512, "temperature": 0.2}, - }) - if err != nil { - return "", err +// processChunk runs the 4-phase enrichment pipeline for a single chunk of docs. +// Phase 1 batches all files into a single LLM call for summarization. +func (w *enrichWorker) processChunk(ctx context.Context, items []eventbus.VaultDocUpsertedPayload) { + // Phase 0 — Prepare: dedup check, batch-fetch existing docs, read file content. + type prepared struct { + payload eventbus.VaultDocUpsertedPayload + content string // non-empty = needs LLM summarization + summary string // non-empty = already summarized, skip LLM + title string // carried from DB for classify phase } - return strings.TrimSpace(resp.Content), nil -} -// autoLink finds similar documents and creates semantic links. -func (w *enrichWorker) autoLink(ctx context.Context, tenantID, agentID, docID string) { - neighbors, err := w.vault.FindSimilarDocs(ctx, tenantID, agentID, docID, enrichSimilarityLimit) - if err != nil { - slog.Warn("vault.enrich: find_similar", "doc", docID, "err", err) + // Filter dedup'd items first. + var pending []eventbus.VaultDocUpsertedPayload + for _, item := range items { + w.dedupMu.Lock() + if prev, exists := w.dedup[item.DocID]; exists && prev == item.ContentHash { + w.dedupMu.Unlock() + continue + } + w.dedupMu.Unlock() + pending = append(pending, item) + } + if len(pending) == 0 { return } - for _, n := range neighbors { - if n.Score < enrichSimilarityMin { + // Batch-fetch all existing docs in a single query. + tenantID := pending[0].TenantID + docIDs := make([]string, len(pending)) + for i, item := range pending { + docIDs[i] = item.DocID + } + existingDocs, err := w.vault.GetDocumentsByIDs(ctx, tenantID, docIDs) + if err != nil { + slog.Warn("vault.enrich: batch_fetch_docs", "count", len(docIDs), "err", err) + return + } + docMap := make(map[string]*store.VaultDocument, len(existingDocs)) + for i := range existingDocs { + docMap[existingDocs[i].ID] = &existingDocs[i] + } + + var all []prepared + for _, item := range pending { + existing := docMap[item.DocID] + if existing != nil && existing.Summary != "" { + all = append(all, prepared{payload: item, summary: existing.Summary, title: existing.Title}) continue } - link := &store.VaultLink{ - FromDocID: docID, - ToDocID: n.Document.ID, - LinkType: "semantic", - Context: fmt.Sprintf("auto-linked (score: %.2f)", n.Score), + if existing != nil && existing.DocType == "media" { + all = append(all, prepared{payload: item, title: existing.Title}) + continue } - if err := w.vault.CreateLink(ctx, link); err != nil { - slog.Debug("vault.enrich: create_link", "from", docID, "to", n.Document.ID, "err", err) + + fullPath := filepath.Join(item.Workspace, item.Path) + raw, err := os.ReadFile(fullPath) + if err != nil { + slog.Warn("vault.enrich: read_file", "path", item.Path, "err", err) + continue + } + runes := []rune(string(raw)) + if len(runes) > enrichBatchItemMaxRunes { + runes = runes[:enrichBatchItemMaxRunes] + } + title := "" + if existing != nil { + title = existing.Title + } + all = append(all, prepared{payload: item, content: string(runes), title: title}) + } + if len(all) == 0 { + return + } + + // Phase 1 — Batch summarize: 1 LLM call for all files needing summary. + var needLLM []int + for i, p := range all { + if p.content != "" && p.summary == "" { + needLLM = append(needLLM, i) } } + if len(needLLM) > 0 { + paths := make([]string, len(needLLM)) + contents := make([]string, len(needLLM)) + for i, idx := range needLLM { + paths[i] = all[idx].payload.Path + contents[i] = all[idx].content + } + summaries := w.batchSummarize(ctx, paths, contents) + for i, idx := range needLLM { + if i < len(summaries) && summaries[i] != "" { + all[idx].summary = summaries[i] + } + } + } + + // Build enriched results. + var results []enriched + for _, p := range all { + results = append(results, enriched{payload: p.payload, summary: p.summary, title: p.title}) + } + + // Phase 2 — Embed: update summary + embed per doc. + var embedded []enriched + for _, r := range results { + if err := w.vault.UpdateSummaryAndReembed(ctx, r.payload.TenantID, r.payload.DocID, r.summary); err != nil { + slog.Warn("vault.enrich: update_summary", "doc", r.payload.DocID, "err", err) + continue + } + embedded = append(embedded, r) + } + + // Phase 3 — Classify links for this chunk. + if len(embedded) > 0 { + first := embedded[0].payload + w.classifyLinks(ctx, first.TenantID, first.AgentID, embedded) + } + + // Phase 4 — Record dedup + wikilinks. + for _, r := range embedded { + w.recordDedup(r.payload.DocID, r.payload.ContentHash) + w.syncWikilinks(ctx, r.payload) + } } +const batchSummarizePrompt = `Summarize each document in 2-3 sentences. Focus on main topic, key concepts, and actionable information. +Output a JSON array: [{"idx":1,"summary":"..."},{"idx":2,"summary":"..."}] +idx is 1-based matching the document number. Output ONLY valid JSON, no preamble.` + +// batchSummarize sends multiple files in a single LLM call and parses JSON summaries. +func (w *enrichWorker) batchSummarize(ctx context.Context, paths, contents []string) []string { + var b strings.Builder + for i := range paths { + fmt.Fprintf(&b, "[%d] File: %s\n%s\n\n", i+1, paths[i], contents[i]) + } + + raw, err := w.chatWithRetry(ctx, "vault.batch_summarize", providers.ChatRequest{ + Messages: []providers.Message{ + {Role: "system", Content: batchSummarizePrompt}, + {Role: "user", Content: b.String()}, + }, + Model: w.model, + Options: map[string]any{"max_tokens": 1536, "temperature": 0.2}, + }) + if err != nil { + slog.Warn("vault.enrich: batch_summarize", "count", len(paths), "err", err) + return nil + } + return parseBatchSummaries(raw, len(paths)) +} + +// parseBatchSummaries extracts summaries from LLM JSON response. +func parseBatchSummaries(raw string, expected int) []string { + raw = strings.TrimSpace(raw) + // Strip markdown code fence if present. + if strings.HasPrefix(raw, "```") { + if i := strings.Index(raw[3:], "\n"); i >= 0 { + raw = raw[3+i+1:] + } + if i := strings.LastIndex(raw, "```"); i >= 0 { + raw = raw[:i] + } + raw = strings.TrimSpace(raw) + } + + var results []struct { + Idx int `json:"idx"` + Summary string `json:"summary"` + } + if err := json.Unmarshal([]byte(raw), &results); err != nil { + slog.Warn("vault.enrich: parse_batch_summaries", "err", err) + return nil + } + + out := make([]string, expected) + for _, r := range results { + if r.Idx >= 1 && r.Idx <= expected { + out[r.Idx-1] = strings.TrimSpace(r.Summary) + } + } + return out +} + +// chatWithRetry is the shared retry loop for all enrichment LLM calls. +// Escalating timeouts and backoffs prevent transient provider failures +// (e.g. 529 overloaded) from permanently skipping documents. +func (w *enrichWorker) chatWithRetry(ctx context.Context, logPrefix string, req providers.ChatRequest) (string, error) { + var lastErr error + for attempt := range enrichMaxRetries { + if attempt > 0 { + select { + case <-ctx.Done(): + return "", ctx.Err() + case <-time.After(enrichRetryBackoffs[attempt]): + } + } + cctx, cancel := context.WithTimeout(ctx, enrichRetryTimeouts[attempt]) + resp, err := w.provider.Chat(cctx, req) + cancel() + if err != nil { + lastErr = err + slog.Warn(logPrefix+": retry", "attempt", attempt+1, "err", err) + continue + } + return strings.TrimSpace(resp.Content), nil + } + return "", fmt.Errorf("%s exhausted %d retries: %w", logPrefix, enrichMaxRetries, lastErr) +} + +// syncWikilinks extracts [[wikilinks]] from document content and syncs them as vault links. +// Skips binary/media files to avoid parsing garbage data as wikilinks. +func (w *enrichWorker) syncWikilinks(ctx context.Context, p eventbus.VaultDocUpsertedPayload) { + doc, err := w.vault.GetDocumentByID(ctx, p.TenantID, p.DocID) + if err != nil || doc == nil { + return + } + if doc.DocType == "media" { + return + } + + fullPath := filepath.Join(p.Workspace, p.Path) + f, err := os.Open(fullPath) + if err != nil { + return + } + // Read up to 4MB — covers large docs while keeping RAM bounded. + buf := make([]byte, 4<<20) + n, _ := f.Read(buf) + f.Close() + if n == 0 { + return + } + content := buf[:n] + + if err := SyncDocLinks(ctx, w.vault, doc, string(content), p.TenantID, p.AgentID); err != nil { + slog.Warn("vault.enrich: sync_wikilinks", "path", p.Path, "err", err) + } +} + + // recordDedup stores a processed hash and evicts ~25% entries if over capacity. func (w *enrichWorker) recordDedup(docID, hash string) { w.dedupMu.Lock() diff --git a/internal/vault/links.go b/internal/vault/links.go index e74cee6e..ce14c62a 100644 --- a/internal/vault/links.go +++ b/internal/vault/links.go @@ -48,7 +48,7 @@ func ExtractWikilinks(content string) []WikilinkMatch { } // ResolveWikilinkTarget finds a vault_document matching the wikilink target. -// Strategy: exact path -> path+.md -> basename match -> nil. +// Strategy: exact path -> path+.md -> basename DB query -> nil. func ResolveWikilinkTarget(ctx context.Context, vs store.VaultStore, target, tenantID, agentID string) (*store.VaultDocument, error) { // 1. Exact path match doc, err := vs.GetDocument(ctx, tenantID, agentID, target) @@ -64,20 +64,17 @@ func ResolveWikilinkTarget(ctx context.Context, vs store.VaultStore, target, ten } } - // 3. Basename search: list all docs and find by basename - docs, err := vs.ListDocuments(ctx, tenantID, agentID, store.VaultListOptions{Limit: 500}) - if err != nil { - return nil, err + // 3. Basename search via targeted DB query (replaces ListDocuments N+1). + targetBase := filepath.Base(target) + doc, err = vs.GetDocumentByBasename(ctx, tenantID, agentID, targetBase) + if err == nil && doc != nil { + return doc, nil } - targetBase := strings.ToLower(filepath.Base(target)) - targetBaseMD := targetBase - if !strings.HasSuffix(targetBaseMD, ".md") { - targetBaseMD = targetBase + ".md" - } - for i := range docs { - base := strings.ToLower(filepath.Base(docs[i].Path)) - if base == targetBase || base == targetBaseMD { - return &docs[i], nil + // Try with .md suffix. + if !strings.HasSuffix(targetBase, ".md") { + doc, err = vs.GetDocumentByBasename(ctx, tenantID, agentID, targetBase+".md") + if err == nil && doc != nil { + return doc, nil } } @@ -89,16 +86,18 @@ func ResolveWikilinkTarget(ctx context.Context, vs store.VaultStore, target, ten func SyncDocLinks(ctx context.Context, vs store.VaultStore, doc *store.VaultDocument, content, tenantID, agentID string) error { matches := ExtractWikilinks(content) if len(matches) == 0 { - // No links — delete existing outbound links - return vs.DeleteDocLinks(ctx, tenantID, doc.ID) + // No wikilinks — delete existing outbound wikilink-type links only. + return vs.DeleteDocLinksByType(ctx, tenantID, doc.ID, "wikilink") } - // Delete old outbound links first (replace strategy) - if err := vs.DeleteDocLinks(ctx, tenantID, doc.ID); err != nil { + // Delete old outbound wikilink-type links first (replace strategy). + // Preserves semantic links created by auto-linking. + if err := vs.DeleteDocLinksByType(ctx, tenantID, doc.ID, "wikilink"); err != nil { return err } - // Resolve and create new links + // Resolve all wikilinks, then batch-create links in a single call. + var links []store.VaultLink for _, m := range matches { target, err := ResolveWikilinkTarget(ctx, vs, m.Target, tenantID, agentID) if err != nil { @@ -109,15 +108,15 @@ func SyncDocLinks(ctx context.Context, vs store.VaultStore, doc *store.VaultDocu slog.Debug("vault.link_unresolved", "target", m.Target) continue } - link := &store.VaultLink{ + links = append(links, store.VaultLink{ FromDocID: doc.ID, ToDocID: target.ID, LinkType: "wikilink", Context: m.Context, - } - if err := vs.CreateLink(ctx, link); err != nil { - slog.Warn("vault.create_link", "from", doc.Path, "to", target.Path, "err", err) - } + }) } - return nil + if len(links) == 0 { + return nil + } + return vs.CreateLinks(ctx, links) } diff --git a/internal/vault/rescan.go b/internal/vault/rescan.go new file mode 100644 index 00000000..6316f63d --- /dev/null +++ b/internal/vault/rescan.go @@ -0,0 +1,216 @@ +package vault + +import ( + "context" + "log/slog" + "path/filepath" + "strings" + "time" + + "github.com/google/uuid" + "github.com/nextlevelbuilder/goclaw/internal/eventbus" + "github.com/nextlevelbuilder/goclaw/internal/store" +) + +// RescanParams holds input for tenant-wide workspace rescan. +// AgentMap and TeamSet are pre-loaded by the caller to avoid per-file DB lookups. +type RescanParams struct { + TenantID string + Workspace string // absolute path to tenant's workspace root + AgentMap map[string]string // agent_key → agent UUID + TeamSet map[string]bool // team UUID → exists (for validation) +} + +// RescanResult holds the outcome of a workspace rescan. +type RescanResult struct { + Scanned int `json:"scanned"` + New int `json:"new"` + Updated int `json:"updated"` + Unchanged int `json:"unchanged"` + Skipped int `json:"skipped"` + Errors int `json:"errors"` + Truncated bool `json:"truncated"` +} + +// RescanWorkspace walks the tenant workspace and registers missing or changed +// files in vault_documents. Ownership (agent/team/scope) is inferred from path. +// Publishes EventVaultDocUpserted for each new or updated file so the +// enrichment worker can process them asynchronously. +func RescanWorkspace(ctx context.Context, params RescanParams, vs store.VaultStore, bus eventbus.DomainEventBus) (*RescanResult, error) { + entries, walkStats, err := SafeWalkWorkspace(ctx, params.Workspace, DefaultWalkOptions()) + if err != nil { + return nil, err + } + + result := &RescanResult{ + Scanned: walkStats.Eligible, + Skipped: walkStats.SkippedExcluded + walkStats.SkippedSymlinks + walkStats.SkippedTooLarge, + Truncated: walkStats.Truncated, + } + + for _, entry := range entries { + agentID, teamID, scope, strippedPath := inferOwnerFromPath(entry.RelPath, params.AgentMap, params.TeamSet) + if scope == "" { + // Unknown agent key or invalid team UUID — skip. + result.Skipped++ + continue + } + + hash, hashErr := ContentHashFile(entry.AbsPath) + if hashErr != nil { + result.Errors++ + continue + } + + // Resolve the agent ID string for store lookup (empty string = no agent filter). + agentIDStr := "" + if agentID != nil { + agentIDStr = *agentID + } + + // Use full workspace-relative path for DB storage and enrichment. + // This preserves the real filesystem path (e.g. "teams/{uuid}/system/file.md") + // so enrichment worker can locate files via filepath.Join(workspace, path). + relPath := entry.RelPath + + // Check if document already exists with same hash. + // Fallback: also check old stripped path (pre-fix records stored without prefix). + existing, _ := vs.GetDocument(ctx, params.TenantID, agentIDStr, relPath) + if existing == nil && strippedPath != relPath { + existing, _ = vs.GetDocument(ctx, params.TenantID, agentIDStr, strippedPath) + } + if existing != nil && existing.ContentHash == hash { + result.Unchanged++ + continue + } + + doc := &store.VaultDocument{ + TenantID: params.TenantID, + AgentID: agentID, + TeamID: teamID, + Scope: scope, + Path: relPath, + Title: InferTitle(relPath), + DocType: InferDocType(relPath), + ContentHash: hash, + } + + if err := vs.UpsertDocument(ctx, doc); err != nil { + slog.Warn("vault.rescan: upsert", "path", relPath, "err", err) + result.Errors++ + continue + } + + if existing != nil { + result.Updated++ + } else { + result.New++ + } + + // Publish enrichment event. AgentID in payload stays string for serialization. + if bus != nil { + bus.Publish(eventbus.DomainEvent{ + ID: uuid.Must(uuid.NewV7()).String(), + Type: eventbus.EventVaultDocUpserted, + SourceID: doc.ID + ":" + hash, + TenantID: params.TenantID, + AgentID: agentIDStr, + Timestamp: time.Now(), + Payload: eventbus.VaultDocUpsertedPayload{ + DocID: doc.ID, + TenantID: params.TenantID, + AgentID: agentIDStr, + Path: relPath, + ContentHash: hash, + Workspace: params.Workspace, + }, + }) + } + } + + slog.Info("vault.rescan", + "tenant", params.TenantID, + "scanned", result.Scanned, "new", result.New, + "updated", result.Updated, "unchanged", result.Unchanged, + "skipped", result.Skipped, "errors", result.Errors, + "truncated", result.Truncated) + + return result, nil +} + +// inferOwnerFromPath parses a tenant-relative path to determine ownership. +// Returns: agentID (*string), teamID (*string), scope (string), strippedPath (string). +// +// Path patterns: +// +// agents/{agent_key}/rest/of/path → agentID=lookup(key), scope="personal", path="rest/of/path" +// teams/{team_uuid}/rest/of/path → teamID=uuid, scope="team", path="rest/of/path" +// anything/else → scope="shared", path unchanged +// +// Returns scope="" to signal the file should be skipped (unknown agent or invalid team). +func inferOwnerFromPath(relPath string, agentMap map[string]string, teamSet map[string]bool) (agentID *string, teamID *string, scope string, strippedPath string) { + switch { + case strings.HasPrefix(relPath, "agents/"): + rest := relPath[len("agents/"):] + key, remainder, hasSlash := strings.Cut(rest, "/") + if !hasSlash || key == "" || strings.Contains(remainder, "..") { + return nil, nil, "", relPath // malformed or path traversal + } + agentUUID, ok := agentMap[key] + if !ok { + return nil, nil, "", relPath // unknown agent key → skip + } + return &agentUUID, nil, "personal", remainder + + case strings.HasPrefix(relPath, "teams/"): + rest := relPath[len("teams/"):] + id, remainder, hasSlash := strings.Cut(rest, "/") + if !hasSlash || id == "" || strings.Contains(remainder, "..") { + return nil, nil, "", relPath // malformed or path traversal + } + if _, parseErr := uuid.Parse(id); parseErr != nil { + return nil, nil, "", relPath // not a valid UUID → skip + } + if !teamSet[id] { + return nil, nil, "", relPath // team not found → skip + } + return nil, &id, "team", remainder + + default: + return nil, nil, "shared", relPath + } +} + +// InferDocType guesses doc_type from path conventions. +// Exported so both rescan and vault interceptor share the same logic. +func InferDocType(relPath string) string { + lower := strings.ToLower(relPath) + ext := strings.ToLower(filepath.Ext(relPath)) + + switch ext { + case ".png", ".jpg", ".jpeg", ".gif", ".webp", ".svg", ".bmp", + ".mp4", ".webm", ".mov", ".avi", ".mkv", + ".mp3", ".wav", ".ogg", ".flac", ".aac", ".m4a": + return "media" + } + + switch { + case strings.HasPrefix(lower, "memory/"): + return "memory" + case strings.Contains(lower, "soul.md") || strings.Contains(lower, "identity.md") || strings.Contains(lower, "agents.md"): + return "context" + case strings.HasPrefix(lower, "skills/") || strings.HasSuffix(lower, "skill.md"): + return "skill" + case strings.HasPrefix(lower, "episodic/"): + return "episodic" + default: + return "note" + } +} + +// InferTitle extracts a human-readable title from a file path. +// Exported so both rescan and vault interceptor share the same logic. +func InferTitle(relPath string) string { + base := filepath.Base(relPath) + return strings.TrimSuffix(base, filepath.Ext(base)) +} diff --git a/internal/vault/rescan_test.go b/internal/vault/rescan_test.go new file mode 100644 index 00000000..2eb2428b --- /dev/null +++ b/internal/vault/rescan_test.go @@ -0,0 +1,160 @@ +package vault + +import "testing" + +// TestInferOwnerFromPath covers the new tenant-wide path parser. +func TestInferOwnerFromPath(t *testing.T) { + agentMap := map[string]string{ + "my-bot": "uuid-1", + "other-bot": "uuid-2", + } + validUUID := "550e8400-e29b-41d4-a716-446655440000" + teamSet := map[string]bool{ + validUUID: true, + } + + tests := []struct { + path string + wantAgentID *string + wantTeamID *string + wantScope string + wantStrippedPath string + }{ + // agents/{key}/... → personal scope, strip prefix + { + path: "agents/my-bot/notes/todo.md", + wantAgentID: strPtr("uuid-1"), + wantScope: "personal", + wantStrippedPath: "notes/todo.md", + }, + // agents/{key} with no trailing path → personal, empty stripped path + { + path: "agents/my-bot/file.md", + wantAgentID: strPtr("uuid-1"), + wantScope: "personal", + wantStrippedPath: "file.md", + }, + // teams/{uuid}/... → team scope, strip prefix + { + path: "teams/" + validUUID + "/doc.md", + wantTeamID: strPtr(validUUID), + wantScope: "team", + wantStrippedPath: "doc.md", + }, + // teams/{uuid}/deep/nested → team scope + { + path: "teams/" + validUUID + "/deep/nested.md", + wantTeamID: strPtr(validUUID), + wantScope: "team", + wantStrippedPath: "deep/nested.md", + }, + // root-level file → shared scope, path unchanged + { + path: "README.md", + wantScope: "shared", + wantStrippedPath: "README.md", + }, + // nested file not under agents/ or teams/ → shared + { + path: "docs/guide.md", + wantScope: "shared", + wantStrippedPath: "docs/guide.md", + }, + // unknown agent → skip (scope="") + { + path: "agents/unknown-bot/file.md", + wantScope: "", + }, + // invalid team UUID → skip + { + path: "teams/not-a-uuid/file.md", + wantScope: "", + }, + // valid UUID but not in teamSet → skip + { + path: "teams/11111111-2222-3333-4444-555555555555/file.md", + wantScope: "", + }, + // malformed agents path (no trailing file) is still an unknown agent key check + { + path: "agents/unknown/", + wantScope: "", + }, + } + + for _, tt := range tests { + t.Run(tt.path, func(t *testing.T) { + gotAgentID, gotTeamID, gotScope, gotPath := inferOwnerFromPath(tt.path, agentMap, teamSet) + + if gotScope != tt.wantScope { + t.Errorf("scope = %q, want %q", gotScope, tt.wantScope) + } + if tt.wantScope == "" { + return // skip is signaled; remaining fields don't matter + } + if tt.wantStrippedPath != "" && gotPath != tt.wantStrippedPath { + t.Errorf("strippedPath = %q, want %q", gotPath, tt.wantStrippedPath) + } + if tt.wantAgentID != nil { + if gotAgentID == nil || *gotAgentID != *tt.wantAgentID { + t.Errorf("agentID = %v, want %q", gotAgentID, *tt.wantAgentID) + } + } else if gotAgentID != nil { + t.Errorf("agentID = %v, want nil", gotAgentID) + } + if tt.wantTeamID != nil { + if gotTeamID == nil || *gotTeamID != *tt.wantTeamID { + t.Errorf("teamID = %v, want %q", gotTeamID, *tt.wantTeamID) + } + } else if gotTeamID != nil { + t.Errorf("teamID = %v, want nil", gotTeamID) + } + }) + } +} + +func TestInferVaultDocType(t *testing.T) { + tests := []struct { + path string + docType string + }{ + {"screenshot.png", "media"}, + {"photo.jpg", "media"}, + {"video.mp4", "media"}, + {"audio.mp3", "media"}, + {"notes/meeting.md", "note"}, + {"report.txt", "note"}, + {"web-fetch/page.html", "note"}, + {"skills/my-skill/SKILL.md", "skill"}, + {"deep/soul.md", "context"}, + } + for _, tt := range tests { + t.Run(tt.path, func(t *testing.T) { + got := InferDocType(tt.path) + if got != tt.docType { + t.Errorf("InferDocType(%q) = %q, want %q", tt.path, got, tt.docType) + } + }) + } +} + +func TestInferTitle(t *testing.T) { + tests := []struct { + path string + title string + }{ + {"report.md", "report"}, + {"notes/meeting-notes.txt", "meeting-notes"}, + {"deep/nested/file.png", "file"}, + } + for _, tt := range tests { + t.Run(tt.path, func(t *testing.T) { + got := InferTitle(tt.path) + if got != tt.title { + t.Errorf("InferTitle(%q) = %q, want %q", tt.path, got, tt.title) + } + }) + } +} + +func strPtr(s string) *string { return &s } diff --git a/internal/vault/safe_walk.go b/internal/vault/safe_walk.go new file mode 100644 index 00000000..a14e3c0a --- /dev/null +++ b/internal/vault/safe_walk.go @@ -0,0 +1,232 @@ +package vault + +import ( + "context" + "errors" + "io/fs" + "os" + "path/filepath" + "strings" + "time" +) + +// WalkEntry represents an eligible file found during workspace walk. +type WalkEntry struct { + RelPath string // workspace-relative, forward-slash separated + AbsPath string // absolute filesystem path + Size int64 // file size in bytes + ModTime time.Time // last modification time +} + +// WalkStats holds metrics from a workspace walk. +type WalkStats struct { + TotalWalked int + Eligible int + SkippedSymlinks int + SkippedExcluded int + SkippedTooLarge int + Truncated bool +} + +// WalkOptions configures resource limits for workspace walking. +type WalkOptions struct { + MaxFiles int // max eligible files to return (0 = unlimited) + MaxTotalBytes int64 // max cumulative file size in bytes (0 = unlimited) + MaxFileBytes int64 // skip individual files larger than this (0 = unlimited) +} + +// DefaultWalkOptions returns safe defaults for production use. +func DefaultWalkOptions() WalkOptions { + return WalkOptions{ + MaxFiles: 5000, + MaxTotalBytes: 500 * 1024 * 1024, // 500MB + MaxFileBytes: 50 * 1024 * 1024, // 50MB per file + } +} + +// SafeWalkWorkspace walks root directory collecting eligible files. +// Symlinks are skipped unconditionally. Excluded paths are filtered. +// Resource limits (file count, total size, context deadline) are enforced. +func SafeWalkWorkspace(ctx context.Context, root string, opts WalkOptions) ([]WalkEntry, WalkStats, error) { + root = filepath.Clean(root) + var ( + entries []WalkEntry + stats WalkStats + totalBytes int64 + walkCount int + ) + + // Early context check before starting walk. + if ctx.Err() != nil { + return nil, stats, ctx.Err() + } + + err := filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error { + if err != nil { + return nil // skip unreadable entries + } + + // Check context periodically. + walkCount++ + if walkCount%100 == 0 { + if ctx.Err() != nil { + return ctx.Err() + } + } + + // Skip ALL symlinks unconditionally (both files and dirs). + if d.Type()&os.ModeSymlink != 0 { + stats.SkippedSymlinks++ + if d.IsDir() { + return filepath.SkipDir + } + return nil + } + + // Compute workspace-relative path (forward-slash). + relPath, relErr := filepath.Rel(root, path) + if relErr != nil || relPath == "." { + return nil + } + relPath = filepath.ToSlash(relPath) + + // Boundary check: path must stay inside root. + if strings.HasPrefix(relPath, "..") { + if d.IsDir() { + return filepath.SkipDir + } + return nil + } + + // Directory-level exclusions: skip entire subtree. + if d.IsDir() { + if isExcludedDir(relPath) { + stats.SkippedExcluded++ + return filepath.SkipDir + } + return nil + } + + stats.TotalWalked++ + + // File-level exclusions. + if isExcludedPath(relPath) { + stats.SkippedExcluded++ + return nil + } + + // File info for size check. + info, infoErr := d.Info() + if infoErr != nil { + return nil // skip unreadable + } + + // Per-file size limit. + if opts.MaxFileBytes > 0 && info.Size() > opts.MaxFileBytes { + stats.SkippedTooLarge++ + return nil + } + + // Total size limit. + if opts.MaxTotalBytes > 0 && totalBytes+info.Size() > opts.MaxTotalBytes { + stats.Truncated = true + return filepath.SkipAll + } + + // File count limit. + if opts.MaxFiles > 0 && len(entries) >= opts.MaxFiles { + stats.Truncated = true + return filepath.SkipAll + } + + totalBytes += info.Size() + stats.Eligible++ + entries = append(entries, WalkEntry{ + RelPath: relPath, + AbsPath: path, + Size: info.Size(), + ModTime: info.ModTime(), + }) + return nil + }) + + // Context cancellation is expected, not an error for partial results. + if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { + return entries, stats, err + } + return entries, stats, err +} + +// isExcludedDir returns true if an entire directory subtree should be skipped. +// relPath is the walk-relative path of the directory (e.g. "agents/my-bot/web-fetch"). +func isExcludedDir(relPath string) bool { + // Get the directory's own name (last segment). + dirName := filepath.Base(relPath) + + // Skip memory/ at any depth. + if dirName == "memory" { + return true + } + + // Skip hidden dirs (. prefix) EXCEPT .uploads (user content). + if strings.HasPrefix(dirName, ".") && dirName != ".uploads" { + return true + } + + // Skip web-fetch/ at any depth — content from external URLs may be dangerous. + if dirName == "web-fetch" { + return true + } + + return false +} + +// contextFiles are root-level bootstrap files managed by ContextFileInterceptor. +var contextFiles = map[string]bool{ + "SOUL.md": true, + "IDENTITY.md": true, + "USER.md": true, + "BOOTSTRAP.md": true, + "AGENTS.md": true, + "TOOLS.md": true, + "CAPABILITIES.md": true, + "MEMORY.md": true, + "AGENTS_CORE.md": true, + "AGENTS_TASK.md": true, +} + +// isExcludedPath returns true if a file should be excluded from vault registration. +// Defense-in-depth: checks ALL parent directory segments for exclusions. +func isExcludedPath(relPath string) bool { + // Check every directory segment in the path. + dir := filepath.Dir(relPath) + for dir != "." && dir != "/" { + seg := filepath.Base(dir) + if seg == "memory" || seg == "web-fetch" { + return true + } + if strings.HasPrefix(seg, ".") && seg != ".uploads" { + return true + } + dir = filepath.Dir(dir) + } + + base := filepath.Base(relPath) + + // SQLite database files. + if strings.HasSuffix(base, ".db") || strings.HasSuffix(base, ".db-wal") || strings.HasSuffix(base, ".db-shm") { + return true + } + + // Root-level context files only (not nested). + if !strings.Contains(relPath, "/") && contextFiles[base] { + return true + } + + // Hidden files at root level. + if strings.HasPrefix(base, ".") && !strings.Contains(relPath, "/") { + return true + } + + return false +} diff --git a/internal/vault/safe_walk_test.go b/internal/vault/safe_walk_test.go new file mode 100644 index 00000000..f9f69b94 --- /dev/null +++ b/internal/vault/safe_walk_test.go @@ -0,0 +1,233 @@ +package vault + +import ( + "context" + "os" + "path/filepath" + "runtime" + "testing" +) + +func TestSafeWalkWorkspace_BasicFiles(t *testing.T) { + dir := t.TempDir() + writeFile(t, dir, "notes/meeting.md", "hello") + writeFile(t, dir, "report.txt", "world") + writeFile(t, dir, "images/screenshot.png", "png-data") + + entries, stats, err := SafeWalkWorkspace(context.Background(), dir, DefaultWalkOptions()) + if err != nil { + t.Fatal(err) + } + if len(entries) != 3 { + t.Fatalf("got %d entries, want 3", len(entries)) + } + if stats.Eligible != 3 { + t.Errorf("stats.Eligible = %d, want 3", stats.Eligible) + } + if stats.Truncated { + t.Error("unexpected truncation") + } +} + +func TestSafeWalkWorkspace_SkipsSymlinks(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("symlinks require elevated privileges on Windows") + } + dir := t.TempDir() + writeFile(t, dir, "real.txt", "data") + // Symlink file + os.Symlink(filepath.Join(dir, "real.txt"), filepath.Join(dir, "link.txt")) + // Symlink dir (pointing outside workspace) + os.Symlink("/tmp", filepath.Join(dir, "escape")) + + entries, stats, err := SafeWalkWorkspace(context.Background(), dir, DefaultWalkOptions()) + if err != nil { + t.Fatal(err) + } + if len(entries) != 1 { + t.Fatalf("got %d entries, want 1 (only real.txt)", len(entries)) + } + if entries[0].RelPath != "real.txt" { + t.Errorf("entry path = %q, want real.txt", entries[0].RelPath) + } + if stats.SkippedSymlinks != 2 { + t.Errorf("stats.SkippedSymlinks = %d, want 2", stats.SkippedSymlinks) + } +} + +func TestSafeWalkWorkspace_ExcludedPaths(t *testing.T) { + dir := t.TempDir() + // Should be excluded + writeFile(t, dir, "memory/session.json", "x") + writeFile(t, dir, ".hidden/file.txt", "x") + writeFile(t, dir, ".wrangler/config.json", "x") + writeFile(t, dir, ".media/thumb.png", "x") + writeFile(t, dir, "SOUL.md", "x") + writeFile(t, dir, "IDENTITY.md", "x") + writeFile(t, dir, "USER.md", "x") + writeFile(t, dir, "BOOTSTRAP.md", "x") + writeFile(t, dir, "AGENTS.md", "x") + writeFile(t, dir, "TOOLS.md", "x") + writeFile(t, dir, "CAPABILITIES.md", "x") + writeFile(t, dir, "MEMORY.md", "x") + writeFile(t, dir, "data.db", "x") + writeFile(t, dir, "data.db-wal", "x") + writeFile(t, dir, "data.db-shm", "x") + writeFile(t, dir, "web-fetch/page.txt", "x") // excluded: external content + // Should NOT be excluded + writeFile(t, dir, "notes/meeting.md", "x") + writeFile(t, dir, ".uploads/photo.jpg", "x") + writeFile(t, dir, "soul-notes.md", "x") + writeFile(t, dir, "deep/SOUL.md", "x") // not root level + writeFile(t, dir, "teams/abc-123/doc.md", "x") + + entries, stats, err := SafeWalkWorkspace(context.Background(), dir, DefaultWalkOptions()) + if err != nil { + t.Fatal(err) + } + if len(entries) != 5 { + names := make([]string, len(entries)) + for i, e := range entries { + names[i] = e.RelPath + } + t.Fatalf("got %d entries %v, want 6 non-excluded", len(entries), names) + } + if stats.SkippedExcluded == 0 { + t.Error("expected some excluded files") + } +} + +func TestSafeWalkWorkspace_MaxFileLimit(t *testing.T) { + dir := t.TempDir() + for i := 0; i < 20; i++ { + writeFile(t, dir, filepath.Join("files", string(rune('a'+i))+".txt"), "data") + } + + opts := DefaultWalkOptions() + opts.MaxFiles = 10 + entries, stats, err := SafeWalkWorkspace(context.Background(), dir, opts) + if err != nil { + t.Fatal(err) + } + if len(entries) > 10 { + t.Fatalf("got %d entries, want <=10", len(entries)) + } + if !stats.Truncated { + t.Error("expected truncated=true") + } +} + +func TestSafeWalkWorkspace_MaxTotalBytes(t *testing.T) { + dir := t.TempDir() + // Create files that exceed total byte limit. + bigContent := make([]byte, 1024) // 1KB each + for i := 0; i < 10; i++ { + writeFile(t, dir, filepath.Join("data", string(rune('a'+i))+".bin"), string(bigContent)) + } + + opts := DefaultWalkOptions() + opts.MaxTotalBytes = 5 * 1024 // 5KB limit + entries, stats, err := SafeWalkWorkspace(context.Background(), dir, opts) + if err != nil { + t.Fatal(err) + } + if len(entries) >= 10 { + t.Fatalf("got %d entries, expected fewer than 10 due to size limit", len(entries)) + } + if !stats.Truncated { + t.Error("expected truncated=true") + } + _ = stats +} + +func TestSafeWalkWorkspace_ContextCancel(t *testing.T) { + dir := t.TempDir() + for i := 0; i < 50; i++ { + writeFile(t, dir, filepath.Join("files", string(rune('a'+i/26))+"_"+string(rune('a'+i%26))+".txt"), "data") + } + + ctx, cancel := context.WithCancel(context.Background()) + cancel() // cancel immediately + + _, _, err := SafeWalkWorkspace(ctx, dir, DefaultWalkOptions()) + if err == nil { + t.Error("expected error from cancelled context") + } +} + +func TestSafeWalkWorkspace_PerFileSizeSkip(t *testing.T) { + dir := t.TempDir() + writeFile(t, dir, "small.txt", "ok") + // Create a file larger than MaxFileBytes + bigContent := make([]byte, 100*1024) // 100KB + writeFile(t, dir, "huge.bin", string(bigContent)) + + opts := DefaultWalkOptions() + opts.MaxFileBytes = 50 * 1024 // 50KB per-file limit + entries, stats, err := SafeWalkWorkspace(context.Background(), dir, opts) + if err != nil { + t.Fatal(err) + } + if len(entries) != 1 { + t.Fatalf("got %d entries, want 1 (only small.txt)", len(entries)) + } + if stats.SkippedTooLarge != 1 { + t.Errorf("stats.SkippedTooLarge = %d, want 1", stats.SkippedTooLarge) + } +} + +func TestIsExcludedPath(t *testing.T) { + tests := []struct { + path string + excluded bool + }{ + {"memory/session-123.json", true}, + {"memory/deep/nested.md", true}, + {".hidden/file.txt", true}, + {".wrangler/config.json", true}, + {".media/thumb.png", true}, + {"SOUL.md", true}, + {"IDENTITY.md", true}, + {"USER.md", true}, + {"BOOTSTRAP.md", true}, + {"AGENTS.md", true}, + {"TOOLS.md", true}, + {"CAPABILITIES.md", true}, + {"MEMORY.md", true}, + {"AGENTS_CORE.md", true}, + {"AGENTS_TASK.md", true}, + {"data.db", true}, + {"data.db-wal", true}, + {"data.db-shm", true}, + // NOT excluded: + {"notes/meeting.md", false}, + {"web-fetch/page.txt", true}, + {"agents/my-bot/web-fetch/data.txt", true}, + {"images/screenshot.png", false}, + {"teams/abc-123/doc.md", false}, + {"soul-notes.md", false}, + {"deep/SOUL.md", false}, // not root-level context file + {".uploads/photo.jpg", false}, + {"report.pdf", false}, + } + for _, tt := range tests { + t.Run(tt.path, func(t *testing.T) { + got := isExcludedPath(tt.path) + if got != tt.excluded { + t.Errorf("isExcludedPath(%q) = %v, want %v", tt.path, got, tt.excluded) + } + }) + } +} + +// writeFile creates a file with the given relative path and content inside dir. +func writeFile(t *testing.T, dir, relPath, content string) { + t.Helper() + abs := filepath.Join(dir, relPath) + if err := os.MkdirAll(filepath.Dir(abs), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(abs, []byte(content), 0o644); err != nil { + t.Fatal(err) + } +} diff --git a/migrations/000045_episodic_recall_tracking.down.sql b/migrations/000045_episodic_recall_tracking.down.sql new file mode 100644 index 00000000..e519415c --- /dev/null +++ b/migrations/000045_episodic_recall_tracking.down.sql @@ -0,0 +1,4 @@ +DROP INDEX IF EXISTS idx_episodic_recall_unpromoted; +ALTER TABLE episodic_summaries DROP COLUMN IF EXISTS last_recalled_at; +ALTER TABLE episodic_summaries DROP COLUMN IF EXISTS recall_score; +ALTER TABLE episodic_summaries DROP COLUMN IF EXISTS recall_count; diff --git a/migrations/000045_episodic_recall_tracking.up.sql b/migrations/000045_episodic_recall_tracking.up.sql new file mode 100644 index 00000000..49adab40 --- /dev/null +++ b/migrations/000045_episodic_recall_tracking.up.sql @@ -0,0 +1,15 @@ +-- Phase 10: dreaming weighted scoring. Track per-episode recall signals +-- that feed ComputeRecallScore() in internal/consolidation/scoring.go. +-- recall_score stores the running-average of search hit scores so the +-- dreaming worker can sort unpromoted entries by perceived usefulness +-- instead of strictly oldest-first. +ALTER TABLE episodic_summaries ADD COLUMN IF NOT EXISTS recall_count INT DEFAULT 0 NOT NULL; +ALTER TABLE episodic_summaries ADD COLUMN IF NOT EXISTS recall_score DOUBLE PRECISION DEFAULT 0 NOT NULL; +ALTER TABLE episodic_summaries ADD COLUMN IF NOT EXISTS last_recalled_at TIMESTAMPTZ; + +-- Partial index for DreamingWorker.ListUnpromotedScored — only touches +-- unpromoted rows and orders by recall_score DESC to match the primary +-- query shape: "top-N unpromoted with highest recall signal". +CREATE INDEX IF NOT EXISTS idx_episodic_recall_unpromoted + ON episodic_summaries(agent_id, user_id, recall_score DESC) + WHERE promoted_at IS NULL; diff --git a/migrations/000046_vault_nullable_agent_id.down.sql b/migrations/000046_vault_nullable_agent_id.down.sql new file mode 100644 index 00000000..4ed96cc0 --- /dev/null +++ b/migrations/000046_vault_nullable_agent_id.down.sql @@ -0,0 +1,18 @@ +-- Revert: delete orphaned docs, restore NOT NULL + CASCADE. +DELETE FROM vault_documents WHERE agent_id IS NULL; + +ALTER TABLE vault_documents ALTER COLUMN agent_id SET NOT NULL; + +ALTER TABLE vault_documents DROP CONSTRAINT vault_documents_agent_id_fkey; +ALTER TABLE vault_documents ADD CONSTRAINT vault_documents_agent_id_fkey + FOREIGN KEY (agent_id) REFERENCES agents(id) ON DELETE CASCADE; + +DROP INDEX IF EXISTS uq_vault_docs_agent_team_scope_path; +CREATE UNIQUE INDEX uq_vault_docs_agent_team_scope_path + ON vault_documents (agent_id, COALESCE(team_id, '00000000-0000-0000-0000-000000000000'), scope, path); + +DROP TRIGGER IF EXISTS trg_vault_docs_agent_null_scope ON vault_documents; +DROP FUNCTION IF EXISTS vault_docs_agent_null_scope_fix(); + +DROP INDEX IF EXISTS idx_vault_docs_agent_scope; +CREATE INDEX idx_vault_docs_agent_scope ON vault_documents(agent_id, scope); diff --git a/migrations/000046_vault_nullable_agent_id.up.sql b/migrations/000046_vault_nullable_agent_id.up.sql new file mode 100644 index 00000000..1ceb2b67 --- /dev/null +++ b/migrations/000046_vault_nullable_agent_id.up.sql @@ -0,0 +1,41 @@ +-- Make agent_id nullable so team-scoped and tenant-shared files can exist +-- without an owning agent. + +-- 1. Drop NOT NULL on agent_id. +ALTER TABLE vault_documents ALTER COLUMN agent_id DROP NOT NULL; + +-- 2. Change FK from CASCADE to SET NULL (agent deletion preserves docs). +ALTER TABLE vault_documents DROP CONSTRAINT vault_documents_agent_id_fkey; +ALTER TABLE vault_documents ADD CONSTRAINT vault_documents_agent_id_fkey + FOREIGN KEY (agent_id) REFERENCES agents(id) ON DELETE SET NULL; + +-- 3. Replace unique index: add tenant_id as leading column, COALESCE both nullable cols. +DROP INDEX IF EXISTS uq_vault_docs_agent_team_scope_path; +CREATE UNIQUE INDEX uq_vault_docs_agent_team_scope_path + ON vault_documents ( + tenant_id, + COALESCE(agent_id, '00000000-0000-0000-0000-000000000000'), + COALESCE(team_id, '00000000-0000-0000-0000-000000000000'), + scope, + path + ); + +-- 4. Trigger: when agent deleted (SET NULL) and no team -> scope='shared'. +CREATE OR REPLACE FUNCTION vault_docs_agent_null_scope_fix() +RETURNS TRIGGER AS $$ +BEGIN + IF NEW.agent_id IS NULL AND OLD.agent_id IS NOT NULL AND NEW.team_id IS NULL THEN + NEW.scope := 'shared'; + END IF; + RETURN NEW; +END; +$$ LANGUAGE plpgsql; + +CREATE TRIGGER trg_vault_docs_agent_null_scope + BEFORE UPDATE OF agent_id ON vault_documents + FOR EACH ROW + EXECUTE FUNCTION vault_docs_agent_null_scope_fix(); + +-- 5. Partial index for agent-scoped queries (skip NULLs). +DROP INDEX IF EXISTS idx_vault_docs_agent_scope; +CREATE INDEX idx_vault_docs_agent_scope ON vault_documents(agent_id, scope) WHERE agent_id IS NOT NULL; diff --git a/migrations/000047_cron_jobs_unique_constraint.down.sql b/migrations/000047_cron_jobs_unique_constraint.down.sql new file mode 100644 index 00000000..307a5510 --- /dev/null +++ b/migrations/000047_cron_jobs_unique_constraint.down.sql @@ -0,0 +1,3 @@ +DROP INDEX IF EXISTS idx_vault_docs_basename; +ALTER TABLE vault_documents DROP COLUMN IF EXISTS path_basename; +DROP INDEX IF EXISTS uq_cron_jobs_agent_tenant_name; diff --git a/migrations/000047_cron_jobs_unique_constraint.up.sql b/migrations/000047_cron_jobs_unique_constraint.up.sql new file mode 100644 index 00000000..c18168d8 --- /dev/null +++ b/migrations/000047_cron_jobs_unique_constraint.up.sql @@ -0,0 +1,19 @@ +-- Deduplicate cron_jobs: keep most recently updated row per (agent_id, tenant_id, name). +DELETE FROM cron_jobs +WHERE id NOT IN ( + SELECT DISTINCT ON (agent_id, tenant_id, name) id + FROM cron_jobs + ORDER BY agent_id, tenant_id, name, updated_at DESC +); + +-- Create unique index (will succeed after dedup). +CREATE UNIQUE INDEX IF NOT EXISTS uq_cron_jobs_agent_tenant_name + ON cron_jobs (agent_id, tenant_id, name); + +-- Generated column for path basename (case-insensitive wikilink resolution). +ALTER TABLE vault_documents ADD COLUMN IF NOT EXISTS path_basename TEXT + GENERATED ALWAYS AS (lower(regexp_replace(path, '.+/', ''))) STORED; + +-- Index for fast basename lookup by tenant. +CREATE INDEX IF NOT EXISTS idx_vault_docs_basename + ON vault_documents(tenant_id, path_basename); diff --git a/pkg/protocol/events.go b/pkg/protocol/events.go index 52ddea7f..3333c134 100644 --- a/pkg/protocol/events.go +++ b/pkg/protocol/events.go @@ -105,6 +105,9 @@ const ( // Tenant access revocation — forces affected user's UI to logout. EventTenantAccessRevoked = "tenant.access.revoked" + + // Vault enrichment pipeline progress. + EventVaultEnrichProgress = "vault.enrich.progress" ) // Agent event subtypes (in payload.type) diff --git a/plans/260406-1304-goclaw-v3-core-design/phase-01-foundation-interfaces.md b/plans/260406-1304-goclaw-v3-core-design/phase-01-foundation-interfaces.md deleted file mode 100644 index e3c693a3..00000000 --- a/plans/260406-1304-goclaw-v3-core-design/phase-01-foundation-interfaces.md +++ /dev/null @@ -1,638 +0,0 @@ -# Phase 1: Foundation Interfaces - -## Context Links - -- Brainstorm: `plans/reports/brainstorm-260406-1059-goclaw-v3-core-redesign.md` (decisions 5, 8, 14, 10) -- Architecture: `plans/reports/Explore-260406-thorough-v3-architecture.md` (sections 1, 2, 6, 7) -- Loop deep dive: `plans/reports/Explore-260406-GoClaw-Agent-Loop-v3-Deep-Dive.md` (section 2, 3) -- Workspace forcing: `plans/reports/Explore-260406-workspace-forcing.md` - -## Overview - -- **Priority:** P1 (all other phases depend on these) -- **Status:** Complete -- **Effort:** 8h design - -4 foundational components with zero cross-dependencies. Can be designed and implemented in parallel. Every subsequent phase (pipeline, memory, integration) depends on at least one of these. - -## Key Insights - -- Token counting currently `chars/4` estimate -- drives ALL compaction/pruning decisions. Replacing with tiktoken-go is highest-ROI change -- Workspace resolution has 5 identified confusion sources (see workspace forcing report). Single immutable WorkspaceContext eliminates dual-key confusion -- EventBus currently `map[string]any` payloads, string topic constants. V3 needs typed events for consolidation pipeline -- Provider interface (`providers.Provider`) is functional but loop handles provider quirks directly. Adapter pattern isolates quirks - ---- - -## A. TokenCounter Interface - -### Interface Contract - -```go -// Package: internal/tokencount - -// TokenCounter provides accurate per-model token counting. -// Replaces chars/4 estimation used in compaction + pruning. -type TokenCounter interface { - // Count returns token count for raw text using model's tokenizer. - Count(model string, text string) int - - // CountMessages returns token count for a message list, - // including per-message overhead (role tokens, separators). - CountMessages(model string, msgs []providers.Message) int - - // ModelContextWindow returns max context tokens for a model. - // Falls back to provider default if model unknown. - ModelContextWindow(model string) int -} - -// TokenizerID identifies which tokenizer a model uses. -type TokenizerID string - -const ( - TokenizerCL100K TokenizerID = "cl100k_base" // Claude, GPT-3.5/4 - TokenizerO200K TokenizerID = "o200k_base" // GPT-4o, GPT-5 - TokenizerFallback TokenizerID = "fallback" // chars/4 -) - -// ModelRegistry maps model name prefixes to tokenizer + context window. -type ModelInfo struct { - TokenizerID TokenizerID - ContextWindow int // default context window for this model -} - -// DefaultRegistry provides built-in model mappings. -// Extend at runtime via RegisterModel(). -var DefaultRegistry = map[string]ModelInfo{ - "claude-": {TokenizerCL100K, 200_000}, - "gpt-4o": {TokenizerO200K, 128_000}, - "gpt-4": {TokenizerCL100K, 128_000}, - "gpt-5": {TokenizerO200K, 1_000_000}, - "qwen-": {TokenizerCL100K, 128_000}, // approx - "deepseek-": {TokenizerCL100K, 128_000}, // approx -} -``` - -### Caching Strategy - -```go -// cachedCounter wraps a tokenizer with per-message hash cache. -// Key: SHA-256(model + role + content + tool_call_ids) -// Value: token count (int) -// Eviction: LRU, 10K entries (typical session < 500 messages) -type cachedCounter struct { - cache *lru.Cache[string, int] // hashicorp/golang-lru/v2 - mu sync.RWMutex -} -``` - -Cache avoids re-counting unchanged messages during mid-loop re-estimation. Only new/modified messages hit tokenizer. - -### Per-Message Overhead - -``` -Per message: +4 tokens (role marker + separators) -System message: +4 tokens additional -Tool calls: name + JSON args counted as text -Tool results: content text counted -Images: model-specific (Claude: tile-based, GPT: similar) -``` - -### Data Flow - -``` -Pipeline stage needs token count - -> TokenCounter.CountMessages(model, messages) - -> For each message: - -> Hash(model, msg) -> check cache - -> Cache miss: tokenizer.Encode(text) -> count -> store - -> Sum + per-message overhead - -> Return total -``` - -### Affected Files (current codebase) - -| File | What changes | -|------|-------------| -| `internal/agent/pruning.go` | Replace `EstimateHistoryTokens()` (chars/4) with `TokenCounter.CountMessages()` | -| `internal/agent/loop_compact.go` | Replace token estimation in compaction trigger | -| `internal/agent/loop.go:352-411` | Mid-loop budget calculation uses token counter | -| `internal/agent/loop_history.go:624-699` | `maybeSummarize()` token estimation | -| `internal/agent/loop_types.go` | `runState.overheadTokens` recalibration logic | - -### Breaking Changes - -- `EstimateHistoryTokens()`, `EstimateTokensWithCalibration()` replaced -- `overheadCalibrated` field in `runState` removed (overhead now computed accurately upfront) -- `pruning.go` constants (`softTrimThreshold`, `hardClearThreshold`) stay but use accurate counts - ---- - -## B. WorkspaceContext - -### Interface Contract - -```go -// Package: internal/workspace - -// Scope defines workspace access boundary. -type Scope string - -const ( - ScopePersonal Scope = "personal" // single user, isolated - ScopeTeam Scope = "team" // team context, shared or isolated - ScopeDelegate Scope = "delegate" // delegated task, scoped access -) - -// WorkspaceContext is resolved ONCE at run start, immutable for the entire run. -// Eliminates dual ctxWorkspace + ctxTeamWorkspace confusion. -type WorkspaceContext struct { - // ActivePath is THE path for all file operations (read/write/list/exec). - // No other workspace path is accessible unless in ReadOnlyPaths. - ActivePath string - - // Scope describes the access boundary type. - Scope Scope - - // ReadOnlyPaths are additional paths the agent can read but NOT write. - // Used for: delegator exports, shared reference dirs. - ReadOnlyPaths []string - - // SharedPath is the shared delegate area (read/write by both delegator + delegatee). - // nil when not in delegation context. - SharedPath *string - - // TeamPath is the team workspace root (nil if not in team context). - TeamPath *string - - // MemoryScope determines memory isolation. - // Defaults to workspace scope. "shared" = all users in agent see same memory. - MemoryScope string - - // KGScope determines knowledge graph isolation. - // Defaults to workspace scope. "shared" = all users in agent see same KG. - KGScope string - - // OwnerID identifies who owns this workspace context (user ID or chat ID). - OwnerID string - - // EnforcementLabel is injected into system prompt verbatim. - // e.g. "ENFORCED -- all file paths resolve here, absolute paths rejected" - EnforcementLabel string -} - -// Resolver produces a WorkspaceContext from request parameters. -// Called once at ContextStage. Result is immutable. -type Resolver interface { - Resolve(ctx context.Context, params ResolveParams) (*WorkspaceContext, error) -} - -// ResolveParams captures all inputs needed to determine workspace. -type ResolveParams struct { - AgentID string - AgentType string // "open" | "predefined" - UserID string - ChatID string - TenantID string - PeerKind string // "direct" | "group" - TeamID *string - TeamConfig *TeamWorkspaceConfig // from teams table - DelegateCtx *DelegateContext // from delegation task - BaseDir string // global workspace root - Sharing *store.WorkspaceSharingConfig -} - -// TeamWorkspaceConfig maps to team.settings JSON. -type TeamWorkspaceConfig struct { - SharedWorkspace bool `json:"shared_workspace"` - WorkspacePath string `json:"workspace_path,omitempty"` -} - -// DelegateContext carries delegation-specific workspace overrides. -type DelegateContext struct { - LinkID string - SharedPath string - ExportPaths []string // read-only exports from delegator -} -``` - -### Resolution Rules - -``` -1. Solo DM, not shared: - ActivePath = {baseDir}/{tenantID}/{agentID}/{userID} - Scope = personal - -2. Solo DM, shared workspace: - ActivePath = {baseDir}/{tenantID}/{agentID} - Scope = personal (shared files, memory still per-user unless overridden) - -3. Team dispatch (member executing task): - ActivePath = {baseDir}/{tenantID}/teams/{teamID}/{chatID} - Scope = team - TeamPath = &ActivePath - -4. Team inbound (leader receiving message): - ActivePath = {baseDir}/{tenantID}/{agentID}/{userID} - Scope = personal - TeamPath = &teamWorkspacePath (read-only reference) - -5. Delegate: - ActivePath = own workspace - Scope = delegate - SharedPath = &sharedDelegatePath - ReadOnlyPaths = delegator exports - -6. Group chat: - ActivePath = {baseDir}/{tenantID}/{agentID}/{chatID} - Scope = personal -``` - -### System Prompt Template Section - -``` -## Workspace - -Active workspace: {{.ActivePath}} ({{.Scope}}) -{{if .EnforcementLabel}}[{{.EnforcementLabel}}]{{end}} -{{if .TeamPath}}Team workspace: {{.TeamPath}} (accessible via team file tools){{end}} -{{if .ReadOnlyPaths}}Read-only paths: {{range .ReadOnlyPaths}}- {{.}} -{{end}}{{end}} - -Context files (SOUL.md, USER.md, MEMORY.md): managed by system, read/write intercepted. -{{if eq .Scope "team"}}This is your active workspace. Personal workspace not accessible during this task.{{end}} -``` - -### Affected Files - -| File | What changes | -|------|-------------| -| `internal/agent/loop_context.go:110-176` | Replace workspace resolution with `Resolver.Resolve()` | -| `internal/tools/context_keys.go` | Remove `ctxWorkspace`, `ctxTeamWorkspace`, `effectiveRestrict()` | -| `internal/tools/filesystem.go:308-340` | `resolvePathWithAllowed()` reads from `WorkspaceContext` | -| `internal/tools/filesystem_write.go` | Same | -| `internal/tools/shell.go:243-267` | Working dir from `WorkspaceContext.ActivePath` | -| `internal/agent/systemprompt.go:498-518` | Replace workspace section with template | -| `internal/agent/systemprompt_sections.go:503-520` | Merge team workspace section into workspace template | - -### Breaking Changes - -- `ctxWorkspace` / `ctxTeamWorkspace` context keys removed -- `effectiveRestrict()` removed (always true, now implicit via WorkspaceContext) -- `WithToolWorkspace()` / `WithToolTeamWorkspace()` replaced by single `WithWorkspaceContext()` -- `tools.WithRestrictToWorkspace()` removed (enforcement = default) - ---- - -## C. EventBus Redesign - -### Interface Contract - -```go -// Package: internal/eventbus (new package, replaces internal/bus for domain events) -// Note: internal/bus MessageBus retained for channel message routing (InboundMessage/OutboundMessage). - -// DomainEvent is a typed event with metadata for consolidation pipeline. -type DomainEvent struct { - ID string // UUID v7 for ordering - Type EventType - SourceID string // dedup key (e.g. session key, run ID) - TenantID string - AgentID string - UserID string - Timestamp time.Time - Payload any // typed per EventType (see below) -} - -// EventType identifies the event category. -type EventType string - -const ( - EventSessionCompleted EventType = "session.completed" - EventEpisodicCreated EventType = "episodic.created" - EventEntityUpserted EventType = "entity.upserted" - EventMemoryLint EventType = "memory.lint" - EventRunCompleted EventType = "run.completed" - EventToolExecuted EventType = "tool.executed" -) - -// Typed payloads -- one per EventType. - -type SessionCompletedPayload struct { - SessionKey string - MessageCount int - TokensUsed int - Summary string // compaction summary if available -} - -type EpisodicCreatedPayload struct { - EpisodicID string - SessionKey string - Summary string - KeyEntities []string // entity names for downstream extraction -} - -type EntityUpsertedPayload struct { - EntityIDs []string // newly upserted entity UUIDs -} - -type MemoryLintPayload struct { - Scope string // "agent" or "user" -} - -type RunCompletedPayload struct { - RunID string - Iterations int - TokensUsed int - ToolCalls int - LoopKilled bool -} - -type ToolExecutedPayload struct { - ToolName string - Duration time.Duration - Success bool - ReadOnly bool -} - -// DomainEventBus manages typed event publishing + worker subscriptions. -type DomainEventBus interface { - // Publish enqueues an event for async processing. - // Non-blocking: returns immediately. - Publish(event DomainEvent) - - // Subscribe registers a worker for a specific event type. - // handler called in worker pool goroutine. - // Returns unsubscribe function. - Subscribe(eventType EventType, handler DomainEventHandler) func() - - // Start launches worker pool. Must be called before Publish. - Start(ctx context.Context) - - // Drain waits for all queued events to be processed. For graceful shutdown. - Drain(timeout time.Duration) error -} - -type DomainEventHandler func(ctx context.Context, event DomainEvent) error -``` - -### Worker Pool Design - -```go -// Config for the domain event bus. -type Config struct { - QueueSize int // buffered channel capacity (default 1000) - WorkerCount int // goroutines per event type (default 2) - RetryAttempts int // retry on handler error (default 3) - RetryDelay time.Duration // backoff base (default 1s, exponential) -} -``` - -Lane-based concurrency: each EventType gets its own worker pool. Events of different types processed in parallel. Events of same type processed sequentially per-worker (but multiple workers can process different events of same type concurrently). - -### Idempotency - -Dedup by `SourceID`: events with same SourceID + EventType processed at most once. Implementation: in-memory set with TTL (1h). If event bus restarts, re-processing is safe because all handlers are idempotent. - -```go -// dedup check before handler dispatch -func (b *bus) dispatch(event DomainEvent) { - key := event.Type + ":" + event.SourceID - if b.seen.Has(key) { - return // already processed - } - b.seen.Set(key, time.Now().Add(b.dedupTTL)) - // dispatch to handler... -} -``` - -### Integration Points - -| Producer | Event | Consumer | -|----------|-------|----------| -| Pipeline ObserveStage | `run.completed` | Metrics, tracing | -| Pipeline ObserveStage | `tool.executed` | Self-evolution metrics | -| Session close / compaction | `session.completed` | Episodic summary worker | -| Episodic worker | `episodic.created` | Semantic extraction worker | -| KG ingest | `entity.upserted` | Dedup check worker | -| Cron scheduler | `memory.lint` | Memory lint worker | - -### Affected Files - -| File | What changes | -|------|-------------| -| `internal/bus/types.go` | Retained for channel routing. DomainEvent types in new package | -| `internal/bus/bus.go` | Retained. New `internal/eventbus/` package alongside | -| `internal/agent/loop_run.go` | Publish `run.completed` after finalize | -| `internal/agent/loop_tools.go` | Publish `tool.executed` per tool call | -| `internal/agent/loop_history.go:624-699` | Publish `session.completed` in `maybeSummarize()` | - -### Breaking Changes - -- None for existing `bus.MessageBus` (retained) -- New `internal/eventbus` package added -- Agent loop gains `DomainEventBus` dependency (injected via `Loop` struct) - ---- - -## D. ProviderAdapter Interface - -### Interface Contract - -```go -// Package: internal/providers (extends existing package) - -// ProviderCapabilities declares what a provider supports. -// Queried by pipeline to choose code paths (streaming vs non-streaming, etc.) -type ProviderCapabilities struct { - Streaming bool // supports ChatStream() - ToolCalling bool // supports tools in request - StreamWithTools bool // can stream while tool calls are in-flight (false for DashScope) - Thinking bool // supports extended thinking / reasoning - Vision bool // supports image inputs - CacheControl bool // supports cache_control blocks (Anthropic) - MaxContextWindow int // default context window for default model - TokenizerID string // for tokencount package mapping -} - -// ProviderAdapter transforms between internal wire format and provider-specific format. -// Each provider implements this to isolate quirks from the pipeline loop. -type ProviderAdapter interface { - // ToRequest converts internal ChatRequest to provider-specific wire bytes. - // Handles: schema transforms, thinking passback, cache control injection. - ToRequest(req ChatRequest) ([]byte, http.Header, error) - - // FromResponse converts provider-specific response bytes to internal ChatResponse. - // Handles: thinking extraction, usage normalization, finish reason mapping. - FromResponse(data []byte) (*ChatResponse, error) - - // FromStreamChunk converts a single SSE chunk to internal StreamChunk. - // Returns nil if chunk should be skipped (keep-alive, metadata). - FromStreamChunk(data []byte) (*StreamChunk, error) - - // Capabilities returns static capability declaration. - Capabilities() ProviderCapabilities - - // Name returns provider identifier. - Name() string -} - -// CapabilitiesAware is optionally implemented by Provider. -// Pipeline checks this to choose code path. -type CapabilitiesAware interface { - Capabilities() ProviderCapabilities -} -``` - -### Adapter Registry - -```go -// AdapterRegistry maps provider names to adapter factories. -// Enables plugin-style provider registration. -type AdapterRegistry struct { - mu sync.RWMutex - adapters map[string]AdapterFactory -} - -type AdapterFactory func(cfg ProviderConfig) (ProviderAdapter, error) - -// Register adds a provider adapter. Called at init() or runtime. -func (r *AdapterRegistry) Register(name string, factory AdapterFactory) { - r.mu.Lock() - defer r.mu.Unlock() - r.adapters[name] = factory -} - -// Get returns adapter for provider name. Error if not registered. -func (r *AdapterRegistry) Get(name string, cfg ProviderConfig) (ProviderAdapter, error) { - r.mu.RLock() - factory, ok := r.adapters[name] - r.mu.RUnlock() - if !ok { - return nil, fmt.Errorf("unknown provider: %s", name) - } - return factory(cfg) -} -``` - -### What Moves to Adapter - -| Currently in loop/provider | Moves to adapter | -|---------------------------|-----------------| -| `RawAssistantContent` passback (Anthropic thinking blocks) | `AnthropicAdapter.ToRequest()` | -| `ThinkingSignature` accumulation | `AnthropicAdapter.FromStreamChunk()` | -| DashScope non-streaming fallback for tool calls | `DashScopeAdapter.Capabilities().StreamWithTools = false` | -| Codex `Phase` field injection | `CodexAdapter.ToRequest()` / `FromResponse()` | -| Schema normalization (tool definitions → provider format) | Each adapter's `ToRequest()` | -| Usage normalization (different field names) | Each adapter's `FromResponse()` | - -### Pipeline Usage - -``` -ThinkStage: - 1. caps := provider.Capabilities() // if CapabilitiesAware - 2. If caps.StreamWithTools == false && len(tools) > 0: - -> use Chat() not ChatStream() - 3. If caps.Thinking: - -> inject thinking options - 4. Call Chat/ChatStream as before (Provider interface unchanged) - 5. Provider internally uses its adapter for wire transforms -``` - -Important: `providers.Provider` interface UNCHANGED. `ProviderAdapter` is internal to each provider implementation. Pipeline queries capabilities via optional `CapabilitiesAware` interface on the provider. - -### Affected Files - -| File | What changes | -|------|-------------| -| `internal/providers/types.go` | Add `ProviderCapabilities`, `CapabilitiesAware` | -| `internal/providers/anthropic.go` | Extract wire transforms to `AnthropicAdapter` | -| `internal/providers/openai.go` | Extract wire transforms to `OpenAIAdapter` | -| `internal/providers/dashscope.go` | `StreamWithTools = false` in capabilities | -| `internal/providers/codex.go` | Phase handling in adapter | -| `internal/agent/loop.go:217-323` | ThinkStage queries capabilities instead of provider-type checks | - -### Breaking Changes - -- `ThinkingCapable` interface superseded by `CapabilitiesAware` (backward compatible -- both can coexist during transition) -- Provider-specific branching in loop.go removed (replaced by capability checks) -- No external API changes - ---- - -## Requirements - -### Functional -- F1: Token counting accurate within 5% of provider-reported usage -- F2: WorkspaceContext resolves all 6 scenarios correctly (solo DM, shared, team dispatch, team inbound, delegate, group) -- F3: DomainEventBus processes events within 100ms p99 (in-process) -- F4: Provider capabilities queryable without LLM call - -### Non-Functional -- NF1: TokenCounter cache hit rate >90% in typical session (messages rarely change) -- NF2: WorkspaceContext immutable after creation (no data races) -- NF3: EventBus handles 1000 events/sec sustained (consolidation pipeline load) -- NF4: All 4 components have zero cross-dependencies (can implement in parallel) - -## Design Steps - -1. Define `TokenCounter` interface + `ModelRegistry` mapping table -2. Implement `cachedCounter` with tiktoken-go + LRU cache -3. Define `WorkspaceContext` struct + `Resolver` interface -4. Implement resolver with resolution rules table -5. Define `DomainEvent` types + `DomainEventBus` interface -6. Implement worker pool with lane-based concurrency + dedup -7. Define `ProviderCapabilities` + `CapabilitiesAware` interface -8. Define `AdapterRegistry` for plugin registration - -## Todo List - -- [x] A1: TokenCounter interface definition — `internal/tokencount/token_counter.go` -- [x] A2: ModelRegistry with prefix-based mapping — `internal/tokencount/token_counter.go` -- [x] A3: Per-message hash cache design — documented in phase, cache impl deferred to implementation -- [x] A4: Fallback strategy for unknown models — `internal/tokencount/fallback_counter.go` -- [x] B1: WorkspaceContext struct definition — `internal/workspace/workspace_context.go` -- [x] B2: Resolver interface + ResolveParams — `internal/workspace/workspace_context.go` -- [x] B3: Resolution rules for all 6 scenarios — documented in phase design -- [x] B4: System prompt template section for workspace — documented in phase design -- [x] B5: Enforcement label design — EnforcementLabel field in WorkspaceContext -- [x] C1: DomainEvent type definitions — `internal/eventbus/event_types.go` -- [x] C2: Typed payload structs for each event — `internal/eventbus/event_types.go` -- [x] C3: DomainEventBus interface — `internal/eventbus/domain_event_bus.go` -- [x] C4: Worker pool config + lane-based concurrency design — `internal/eventbus/domain_event_bus.go` -- [x] C5: Idempotency / dedup strategy — documented in phase design (SourceID-based) -- [x] D1: ProviderCapabilities struct — `internal/providers/capabilities.go` -- [x] D2: CapabilitiesAware optional interface — `internal/providers/capabilities.go` -- [x] D3: AdapterRegistry for plugin registration — `internal/providers/adapter_registry.go` -- [x] D4: Document what moves from loop to each adapter — documented in phase design - -## Success Criteria - -- All 4 interfaces defined with complete method signatures -- Each interface has zero imports from the other 3 -- WorkspaceContext resolution rules cover all 6 scenarios -- Token counting accuracy validated against Anthropic/OpenAI reported usage -- EventBus worker pool supports lane-based concurrency with configurable workers -- Provider capabilities capture all known provider quirks (DashScope, Codex, Anthropic) - -## Risk Assessment - -| Risk | Severity | Mitigation | -|------|----------|------------| -| tiktoken-go tokenizer not exact match for Claude models | MEDIUM | Claude uses custom tokenizer, cl100k_base is approximation. Validate against API usage. Accept 5% error. | -| WorkspaceContext missing edge case for new workspace scenario | LOW | Resolution rules table is explicit. Add new row when new scenario appears. | -| EventBus in-process limitation (single instance, no persistence) | LOW | Sufficient for v3.0. External bus (Redis streams) can implement same interface later. | -| ProviderAdapter abstraction leaks (provider needs loop context) | MEDIUM | Keep adapter stateless. Pass all needed data via `ChatRequest` fields. | - -## Security Considerations - -- WorkspaceContext enforcement is security-critical: `ActivePath` must be validated (no path traversal) -- Existing `resolvePath()` security (symlink, hardlink checks) preserved -- WorkspaceContext provides the boundary, not the validation -- EventBus handlers must not leak tenant data across boundaries (TenantID in every event) -- TokenCounter is read-only, no security concerns - -## Next Steps - -- Phase 2 (Pipeline Loop) consumes all 4 interfaces -- Phase 3 (Memory & KG) consumes TokenCounter + DomainEventBus -- Phase 4 (System Integration) consumes WorkspaceContext + ProviderCapabilities diff --git a/plans/260406-1304-goclaw-v3-core-design/phase-02-pipeline-loop-design.md b/plans/260406-1304-goclaw-v3-core-design/phase-02-pipeline-loop-design.md deleted file mode 100644 index 1c18e283..00000000 --- a/plans/260406-1304-goclaw-v3-core-design/phase-02-pipeline-loop-design.md +++ /dev/null @@ -1,653 +0,0 @@ -# Phase 2: Pipeline Loop Design - -## Context Links - -- Brainstorm: `plans/reports/brainstorm-260406-1059-goclaw-v3-core-redesign.md` (decision 1) -- Loop deep dive: `plans/reports/Explore-260406-GoClaw-Agent-Loop-v3-Deep-Dive.md` (all sections) -- Phase 1 dependencies: `phase-01-foundation-interfaces.md` (TokenCounter, WorkspaceContext, EventBus, ProviderCapabilities) -- Current loop: `internal/agent/loop.go` (804 lines), `loop_types.go`, `loop_context.go`, `loop_history.go`, `loop_compact.go`, `pruning.go`, `loop_tools.go`, `loop_run.go` - -## Overview - -- **Priority:** P1 (core execution engine, blocks phases 3-6) -- **Status:** Complete -- **Effort:** 8h design -- **Depends on:** Phase 1 (all 4 foundation interfaces) - -Replace monolithic `runLoop()` (804 lines, 20+ field `runState`) with a pipeline of pluggable stages. Each stage owns a substage of state, is independently testable, and communicates via a shared `RunState` struct. - -## Key Insights - -- Current loop has 7 exit conditions scattered across 800 lines. Pipeline model makes each exit condition a stage responsibility. -- `runState` has 20+ fields mixing concerns (loop control, output accumulation, crash safety, bootstrap detection, team task tracking, skill evolution). Split into typed substates. -- Mid-loop compaction (lines 352-411) interleaves with think phase. Pipeline separates this into `PruneStage` between think and tool. -- Parallel tool execution already well-isolated (goroutine per tool, results collected via channel). Wrapping in `ToolStage` is clean. -- Memory flush currently triggers inside `maybeSummarize()` which runs post-loop. Moving to `MemoryFlushStage` pre-compaction aligns with 3-tier memory design. - ---- - -## A. Stage Interface - -```go -// Package: internal/pipeline - -// Stage is a single step in the agent pipeline. -// Stages are stateless -- all mutable state lives in RunState. -type Stage interface { - // Name returns a human-readable identifier for logging/tracing. - Name() string - - // Execute performs the stage's work. Returns error to abort pipeline. - // Stages modify RunState in place. - Execute(ctx context.Context, state *RunState) error -} - -// StageResult signals how the pipeline should proceed after a stage. -type StageResult int - -const ( - Continue StageResult = iota // proceed to next stage - BreakLoop // exit iteration loop (normal completion) - AbortRun // abort entire run (error/kill) -) - -// StageWithResult extends Stage to control pipeline flow. -// If a stage does not implement this, pipeline assumes Continue. -type StageWithResult interface { - Stage - Result() StageResult -} -``` - -## B. RunState Redesign - -Split current 20+ field `runState` into typed substates. Each substate grouped by owning stage. - -```go -// RunState is the shared mutable state for a single pipeline run. -// Passed by pointer through all stages. -type RunState struct { - // Identity (set once at pipeline start, immutable during run) - Request *agent.RunRequest - Workspace *workspace.WorkspaceContext - Model string - Provider providers.Provider - - // Message buffer (read/write by multiple stages) - Messages *MessageBuffer - - // Per-stage substates - Context ContextState - Think ThinkState - Prune PruneState - Tool ToolState - Observe ObserveState - Compact CompactState - Evolution EvolutionState - - // Cross-cutting concerns - Iteration int - RunID string - ExitCode StageResult -} - -// MessageBuffer wraps the message list with append/replace semantics. -// Thread-safe for read; write only from owning stage (sequential pipeline). -type MessageBuffer struct { - system providers.Message // system prompt (rebuilt by ContextStage) - history []providers.Message // conversation history - pending []providers.Message // new messages this run (flushed at checkpoint) -} - -func (mb *MessageBuffer) All() []providers.Message { /* system + history + pending */ } -func (mb *MessageBuffer) AppendPending(msg providers.Message) { /* ... */ } -func (mb *MessageBuffer) FlushPending() []providers.Message { /* move pending to history */ } -func (mb *MessageBuffer) ReplaceHistory(msgs []providers.Message) { /* post-compaction */ } -``` - -### Substates - -```go -// ContextState: owned by ContextStage, read by ThinkStage -type ContextState struct { - ContextFiles []bootstrap.ContextFile - SkillsSummary string - TeamContext *TeamContextInfo // nil if not in team - HadBootstrap bool - OverheadTokens int // system prompt + context files token count (accurate via TokenCounter) -} - -// ThinkState: owned by ThinkStage -type ThinkState struct { - LastResponse *providers.ChatResponse - TotalUsage providers.Usage - TruncRetries int // consecutive truncation retries (max 3) - StreamingActive bool // true during active stream -} - -// PruneState: owned by PruneStage -type PruneState struct { - MidLoopCompacted bool // true after first in-loop compaction - HistoryTokens int // last computed history token count - HistoryBudget int // contextWindow * maxHistoryShare -} - -// ToolState: owned by ToolStage -type ToolState struct { - LoopDetector *ToolLoopDetector - TotalToolCalls int - AsyncToolCalls []string // tool names that executed async (spawn) - MediaResults []MediaResult // files produced by tools - Deliverables []string // content for team task results - LoopKilled bool // set when loop detector triggers critical -} - -// ObserveState: owned by ObserveStage -type ObserveState struct { - FinalContent string // accumulated response text - FinalThinking string // reasoning output - BlockReplies int - LastBlockReply string -} - -// CompactState: owned by CheckpointStage + MemoryFlushStage -type CompactState struct { - CheckpointFlushedMsgs int - MemoryFlushedThisCycle bool - CompactionCount int // tracks compaction cycles -} - -// EvolutionState: owned by skill evolution nudge logic -type EvolutionState struct { - Nudge70Sent bool - Nudge90Sent bool - PostscriptSent bool - BootstrapWrite bool // BOOTSTRAP.md write detected - TeamTaskCreates int - TeamTaskSpawns int -} -``` - -### Migration from current runState - -| Current field | Moves to | -|--------------|----------| -| `loopDetector` | `ToolState.LoopDetector` | -| `totalUsage` | `ThinkState.TotalUsage` | -| `iteration` | `RunState.Iteration` | -| `totalToolCalls` | `ToolState.TotalToolCalls` | -| `finalContent` | `ObserveState.FinalContent` | -| `finalThinking` | `ObserveState.FinalThinking` | -| `asyncToolCalls` | `ToolState.AsyncToolCalls` | -| `mediaResults` | `ToolState.MediaResults` | -| `deliverables` | `ToolState.Deliverables` | -| `pendingMsgs` | `MessageBuffer.pending` | -| `blockReplies` | `ObserveState.BlockReplies` | -| `lastBlockReply` | `ObserveState.LastBlockReply` | -| `checkpointFlushedMsgs` | `CompactState.CheckpointFlushedMsgs` | -| `midLoopCompacted` | `PruneState.MidLoopCompacted` | -| `overheadTokens` | `ContextState.OverheadTokens` | -| `overheadCalibrated` | REMOVED (overhead now accurate upfront via TokenCounter) | -| `bootstrapWriteDetected` | `EvolutionState.BootstrapWrite` | -| `teamTaskCreates/Spawns` | `EvolutionState.TeamTaskCreates/Spawns` | -| `skillNudge*` | `EvolutionState.Nudge*` | -| `loopKilled` | `ToolState.LoopKilled` | -| `truncationRetries` | `ThinkState.TruncRetries` | - ---- - -## C. Stage Catalog - -### Setup Pipeline (runs once before iteration loop) - -``` -1. ContextStage - - Resolve WorkspaceContext via Resolver - - Load + filter context files (bootstrap, per-user, team) - - Build skills summary (BM25) - - Build system prompt via template - - Compute overhead tokens (TokenCounter) - - Enrich input media - - Inject team task reminders - - Reads: RunState.Request, providers - Writes: RunState.Workspace, ContextState, MessageBuffer.system - - Replaces: injectContext() + buildMessages() + enrichInputMedia() + injectTeamTaskReminders() -``` - -### Iteration Pipeline (runs per iteration, max N times) - -``` -2. ThinkStage - - Inject iteration budget nudges (70%, 90%) - - Build per-iteration tool definitions (policy + bootstrap filters) - - Build ChatRequest from MessageBuffer - - Call LLM (Chat or ChatStream based on capabilities) - - Emit events: chunk, thinking - - Accumulate token usage - - Handle truncation retries - - Reads: MessageBuffer, ContextState, ProviderCapabilities - Writes: ThinkState, MessageBuffer (append assistant message) - Result: Continue (has tool calls) | BreakLoop (no tool calls, final content) - - Replaces: loop.go lines 217-446 - -3. PruneStage - - Compute history tokens via TokenCounter - - Phase 1 @ 70% budget: pruneContextMessages (soft-trim + hard-clear) - - Phase 2 @ 100% budget: compactMessagesInPlace (LLM summarization) - - If still over budget after Phase 2: AbortRun - - Reads: MessageBuffer, TokenCounter, context window - Writes: PruneState, MessageBuffer (may replace history) - Result: Continue | AbortRun (budget exceeded after compaction) - - Replaces: loop.go lines 352-411, loop_compact.go, pruning.go - -4. ToolStage - - Extract tool calls from ThinkState.LastResponse - - Generate unique tool call IDs - - Single tool: sequential execution - - Multiple tools: parallel goroutines, collect via channel, sort by index - - Per-tool: policy check, lazy MCP activate, execute, process result - - Loop detection: same-tool, read-only streak, same-result - - Tool budget check (totalToolCalls > maxToolCalls) - - Emit events: tool.call, tool.result - - Reads: ThinkState.LastResponse, ToolExecutor, WorkspaceContext - Writes: ToolState, MessageBuffer (append tool results) - Result: Continue | BreakLoop (loop killed) | AbortRun (policy denied) - - Replaces: loop.go lines 450-738, loop_tools.go - -5. ObserveStage - - Process tool results: extract deliverables, media - - Drain InjectCh (mid-run message injection, Point A + Point B) - - Check if final content ready (no tool calls + no injected messages) - - Sanitize final content - - Skill evolution postscript - - Publish domain events: tool.executed, run metrics - - Reads: ToolState, ThinkState, InjectCh - Writes: ObserveState, MessageBuffer (may append injected messages) - Result: Continue (more iterations needed) | BreakLoop (final content ready) - - Replaces: loop.go lines 450-466 (no-tool path), finalize portions - -6. CheckpointStage - - Every N iterations (default 5): flush pending messages to session store - - Crash safety: intermediate state persisted - - Conditional: skip if iteration % N != 0 - - Reads: MessageBuffer.pending, SessionStore - Writes: CompactState.CheckpointFlushedMsgs, MessageBuffer (clear pending) - - Replaces: loop.go lines 748-761 - -7. MemoryFlushStage - - Runs before compaction (when PruneStage detects budget pressure) - - Triggered by PruneStage setting a flag, NOT by iteration count - - Two-stage fallback: LLM flush -> extractive regex - - Dedup guard: once per compaction cycle - - Publishes: session.completed event (for episodic pipeline) - - Reads: MessageBuffer, MemoryStore, compaction config - Writes: CompactState.MemoryFlushedThisCycle - - Replaces: memoryflush.go, extractive_memory.go -``` - -### Post-Loop Pipeline (runs once after iteration loop exits) - -``` -8. FinalizeStage - - Sanitize final content (HTML entities, markdown cleanup) - - Media dedup + size population - - Flush remaining pending messages to session - - Token accumulation to session metadata - - Bootstrap auto-cleanup (delete BOOTSTRAP.md if write detected) - - Post-run summarization (maybeSummarize) - - Publish run.completed event - - Construct RunResult - - Reads: All substates - Writes: RunResult (returned to caller) - - Replaces: finalizeRun(), maybeSummarize() -``` - ---- - -## D. Pipeline Orchestrator - -```go -// Pipeline orchestrates stage execution for a single agent run. -type Pipeline struct { - setup []Stage // runs once before loop - iteration []Stage // runs per iteration - finalize []Stage // runs once after loop - - maxIterations int - tokenCounter tokencount.TokenCounter - eventBus eventbus.DomainEventBus -} - -// NewDefaultPipeline creates the standard agent pipeline. -func NewDefaultPipeline(deps PipelineDeps) *Pipeline { - return &Pipeline{ - setup: []Stage{ - NewContextStage(deps), - }, - iteration: []Stage{ - NewThinkStage(deps), - NewPruneStage(deps), - NewToolStage(deps), - NewObserveStage(deps), - NewCheckpointStage(deps), - }, - finalize: []Stage{ - NewFinalizeStage(deps), - }, - maxIterations: deps.Config.MaxIterations, - tokenCounter: deps.TokenCounter, - eventBus: deps.EventBus, - } -} - -// PipelineDeps bundles all dependencies injected into stages. -type PipelineDeps struct { - Provider providers.Provider - TokenCounter tokencount.TokenCounter - EventBus eventbus.DomainEventBus - SessionStore store.SessionStore - ToolExecutor tools.ToolExecutor - ToolPolicy *tools.PolicyEngine - SkillsLoader *skills.Loader - MemoryStore store.MemoryStore - WorkspaceResolver workspace.Resolver - Config PipelineConfig -} - -type PipelineConfig struct { - MaxIterations int - MaxToolCalls int - CheckpointInterval int // flush every N iterations (default 5) - ContextWindow int - MaxTokens int - Compaction *config.CompactionConfig - ContextPruning *config.ContextPruningConfig -} -``` - -### Execution Flow - -```go -// Run executes the full pipeline for a single agent run. -func (p *Pipeline) Run(ctx context.Context, req *agent.RunRequest) (*agent.RunResult, error) { - state := NewRunState(req) - - // 1. Setup pipeline (once) - for _, stage := range p.setup { - if err := stage.Execute(ctx, state); err != nil { - return nil, fmt.Errorf("setup %s: %w", stage.Name(), err) - } - } - - // 2. Iteration loop - for state.Iteration = 0; state.Iteration < p.maxIterations; state.Iteration++ { - for _, stage := range p.iteration { - if err := stage.Execute(ctx, state); err != nil { - return nil, fmt.Errorf("iter %d %s: %w", state.Iteration, stage.Name(), err) - } - - // Check if stage wants to control flow - if swr, ok := stage.(StageWithResult); ok { - switch swr.Result() { - case BreakLoop: - goto finalize - case AbortRun: - goto finalize - } - } - } - - // Check context cancellation (/stop) - if ctx.Err() != nil { - break - } - } - -finalize: - // 3. Finalize pipeline (once) - for _, stage := range p.finalize { - if err := stage.Execute(ctx, state); err != nil { - slog.Warn("finalize stage error", "stage", stage.Name(), "err", err) - // finalize errors are logged, not fatal - } - } - - return state.BuildResult(), nil -} -``` - -### Error Propagation - -``` -Stage returns error: - - Setup stage error -> abort run, return error - - Iteration stage error -> abort run, still run finalize - - Finalize stage error -> log warning, continue other finalize stages - -Stage returns StageResult: - - Continue -> next stage in iteration - - BreakLoop -> skip remaining iteration stages, jump to finalize - - AbortRun -> skip remaining stages, jump to finalize (with error flag) -``` - ---- - -## Data Flow Between Stages - -``` -ContextStage - writes: Workspace, ContextState, MessageBuffer.system - | - v -ThinkStage - reads: MessageBuffer.All(), ContextState.OverheadTokens - writes: ThinkState.LastResponse, ThinkState.TotalUsage - writes: MessageBuffer.AppendPending(assistant msg) - | - v -PruneStage - reads: MessageBuffer, TokenCounter, PruneState - writes: PruneState.HistoryTokens - may trigger: MemoryFlushStage (inline, before compaction) - may write: MessageBuffer.ReplaceHistory(compacted) - | - v -ToolStage - reads: ThinkState.LastResponse.ToolCalls - writes: ToolState.*, MessageBuffer.AppendPending(tool results) - | - v -ObserveStage - reads: ToolState, ThinkState, InjectCh - writes: ObserveState.FinalContent (if done) - may write: MessageBuffer.AppendPending(injected msgs) - | - v -CheckpointStage - reads: MessageBuffer.pending - writes: SessionStore, CompactState.CheckpointFlushedMsgs - | - v (loop back to ThinkStage or exit to FinalizeStage) - -FinalizeStage - reads: All substates - writes: RunResult -``` - ---- - -## MemoryFlushStage Integration with PruneStage - -MemoryFlushStage is NOT a regular iteration stage. It is invoked inline by PruneStage when compaction is needed. - -```go -// PruneStage.Execute -func (s *PruneStage) Execute(ctx context.Context, state *RunState) error { - tokens := s.tokenCounter.CountMessages(state.Model, state.Messages.All()) - state.Prune.HistoryTokens = tokens - - budget := s.contextWindow * 80 / 100 // 80% for history - - // Phase 1: soft pruning at 70% - if tokens > budget*70/100 { - pruneContextMessages(state.Messages, s.config) - tokens = s.tokenCounter.CountMessages(state.Model, state.Messages.All()) - } - - // Phase 2: full compaction at 100% - if tokens > budget { - // Memory flush BEFORE compaction (saves memories before they're summarized away) - if !state.Compact.MemoryFlushedThisCycle && s.memoryFlush != nil { - s.memoryFlush.Execute(ctx, state) // inline call, not pipeline stage - state.Compact.MemoryFlushedThisCycle = true - } - - state.Prune.MidLoopCompacted = true - compactMessagesInPlace(state.Messages, s.compactionConfig) - - // Re-check after compaction - tokens = s.tokenCounter.CountMessages(state.Model, state.Messages.All()) - if tokens > budget { - state.ExitCode = AbortRun - } - } - return nil -} -``` - ---- - -## Requirements - -### Functional -- F1: Pipeline produces identical `RunResult` as current monolithic loop for same inputs -- F2: All 7 exit conditions preserved: no-tool-calls, max iterations, truncation limit, loop kill, read-only streak, tool budget, ctx.Done -- F3: Parallel tool execution behavior unchanged (goroutine per tool, sorted results) -- F4: Mid-run injection (InjectCh) works at both Point A (after tools) and Point B (no tools) -- F5: Checkpoint flush every N iterations, crash safety preserved - -### Non-Functional -- NF1: No measurable latency increase from stage dispatch overhead (stages are coarse-grained) -- NF2: Each stage independently testable with mock RunState -- NF3: Adding new stage requires only: implement Stage interface + insert in pipeline constructor -- NF4: RunState substates compile-time typed (no `map[string]any`) - -## Related Code Files - -### Files to decompose - -| Current file | Lines | Maps to stages | -|-------------|-------|---------------| -| `internal/agent/loop.go` | 804 | ThinkStage, ToolStage, ObserveStage, orchestrator | -| `internal/agent/loop_run.go` | 245 | Pipeline.Run() entry point | -| `internal/agent/loop_context.go` | 275 | ContextStage | -| `internal/agent/loop_history.go` | 859 | ContextStage (buildMessages), FinalizeStage (maybeSummarize) | -| `internal/agent/loop_compact.go` | 119 | PruneStage | -| `internal/agent/pruning.go` | 327 | PruneStage | -| `internal/agent/loop_tools.go` | ~180 | ToolStage | -| `internal/agent/loop_tool_filter.go` | ~100 | ThinkStage (buildFilteredTools) | -| `internal/agent/memoryflush.go` | ~150 | MemoryFlushStage | -| `internal/agent/extractive_memory.go` | ~150 | MemoryFlushStage (fallback) | -| `internal/agent/loop_types.go` | ~150 | RunState, substates | -| `internal/agent/loop_input_media.go` | ~100 | ContextStage | -| `internal/agent/loop_team_reminders.go` | ~60 | ContextStage | - -### New files to create - -| File | Purpose | -|------|---------| -| `internal/pipeline/stage.go` | Stage interface, StageResult, StageWithResult | -| `internal/pipeline/run_state.go` | RunState + MessageBuffer + all substates | -| `internal/pipeline/pipeline.go` | Pipeline orchestrator | -| `internal/pipeline/context_stage.go` | ContextStage implementation | -| `internal/pipeline/think_stage.go` | ThinkStage implementation | -| `internal/pipeline/prune_stage.go` | PruneStage + inline MemoryFlush | -| `internal/pipeline/tool_stage.go` | ToolStage implementation | -| `internal/pipeline/observe_stage.go` | ObserveStage implementation | -| `internal/pipeline/checkpoint_stage.go` | CheckpointStage implementation | -| `internal/pipeline/finalize_stage.go` | FinalizeStage implementation | - -## Design Steps - -1. Define Stage interface + StageResult enum -2. Design RunState with typed substates (map current fields) -3. Define MessageBuffer with append/replace/flush semantics -4. Design each stage's Execute signature and data flow -5. Design Pipeline orchestrator with setup/iteration/finalize phases -6. Map error propagation rules (setup fatal, iteration -> finalize, finalize logged) -7. Design MemoryFlushStage inline integration with PruneStage -8. Verify all 7 exit conditions map to StageResult values - -## Todo List - -- [x] A1: Stage interface + StageResult + StageWithResult — `internal/pipeline/stage.go` -- [x] B1: RunState top-level struct — `internal/pipeline/run_state.go` -- [x] B2: MessageBuffer with thread-safe read, owning-stage write — `internal/pipeline/run_state.go` -- [x] B3: ContextState substate — `internal/pipeline/substates.go` -- [x] B4: ThinkState substate — `internal/pipeline/substates.go` -- [x] B5: PruneState substate — `internal/pipeline/substates.go` -- [x] B6: ToolState substate — `internal/pipeline/substates.go` -- [x] B7: ObserveState substate — `internal/pipeline/substates.go` -- [x] B8: CompactState substate — `internal/pipeline/substates.go` -- [x] B9: EvolutionState substate — `internal/pipeline/substates.go` -- [x] C1: ContextStage design — documented in phase (impl deferred) -- [x] C2: ThinkStage design — documented in phase (impl deferred) -- [x] C3: PruneStage design — documented in phase (impl deferred) -- [x] C4: ToolStage design — documented in phase (impl deferred) -- [x] C5: ObserveStage design — documented in phase (impl deferred) -- [x] C6: CheckpointStage design — documented in phase (impl deferred) -- [x] C7: MemoryFlushStage design — documented in phase (impl deferred) -- [x] C8: FinalizeStage design — documented in phase (impl deferred) -- [x] D1: Pipeline orchestrator with 3-phase execution — `internal/pipeline/pipeline.go` -- [x] D2: PipelineDeps dependency bundle — via NewPipeline() params -- [x] D3: Error propagation rules — setup fatal, iteration fatal+finalize, finalize logged -- [x] E1: Verify all 7 exit conditions preserved — documented in phase design -- [x] E2: Map current runState fields to substates — documented in phase design - -## Success Criteria - -- Every line of current `loop.go` maps to exactly one stage -- `runState` fields fully accounted for in substates (no orphans) -- Pipeline.Run() produces same RunResult semantics as current Loop.Run() -- Each stage can be tested with a constructed RunState (no Loop struct dependency) -- MemoryFlushStage integrated with PruneStage without circular dependency -- InjectCh drain preserved at both injection points - -## Risk Assessment - -| Risk | Severity | Mitigation | -|------|----------|------------| -| Subtle behavior change from stage ordering | HIGH | Integration tests comparing v2 vs v3 output for same inputs. Run existing test suite. | -| InjectCh timing sensitivity (Point A vs Point B) | MEDIUM | ObserveStage handles both points explicitly. Document injection semantics. | -| MemoryFlushStage inline call breaks stage isolation | LOW | Acceptable trade-off: flush MUST happen before compaction. Document as intentional coupling. | -| RunState too large (sum of substates) | LOW | Substates are small structs. Total < 500 bytes. No heap pressure. | -| Stage interface too simple for complex stages | LOW | StageWithResult extension handles flow control. Further extension possible via StageWithCleanup etc. | - -## Security Considerations - -- ToolStage must preserve all existing security checks: policy engine, shell deny patterns, workspace restriction -- ContextStage must preserve input guard scanning (injection pattern detection) -- CheckpointStage flushes to session store with existing tenant scoping -- No new attack surface introduced by pipeline refactor - -## Next Steps - -- Phase 3 (Memory & KG) adds MemoryFlushStage consumer for `session.completed` events -- Phase 4 (System Integration) adds template-based system prompt in ContextStage -- Phase 5 (Orchestration) may add DelegateStage or modify ToolStage for delegation diff --git a/plans/260406-1304-goclaw-v3-core-design/phase-03-memory-kg-design.md b/plans/260406-1304-goclaw-v3-core-design/phase-03-memory-kg-design.md deleted file mode 100644 index 19def25a..00000000 --- a/plans/260406-1304-goclaw-v3-core-design/phase-03-memory-kg-design.md +++ /dev/null @@ -1,797 +0,0 @@ -# Phase 3: Memory & KG Design - -## Context Links - -- Brainstorm: `plans/reports/brainstorm-260406-1059-goclaw-v3-core-redesign.md` (decisions 2, 3, 4, 6, 7) -- Research: `plans/reports/researcher-260406-1059-memory-context-kg-research.md` -- Architecture: `plans/reports/Explore-260406-thorough-v3-architecture.md` (sections 1, 2, 8, 10) -- Phase 1: `phase-01-foundation-interfaces.md` (TokenCounter, DomainEventBus) -- Phase 2: `phase-02-pipeline-loop-design.md` (PruneStage, MemoryFlushStage, ObserveStage) -- Current memory store: `internal/store/memory_store.go` -- Current KG store: `internal/store/knowledge_graph_store.go` -- Current KG migration: `migrations/000013_knowledge_graph.up.sql` -- Current memory migration: `migrations/000001_init_schema.up.sql` (lines 163-192) - -## Overview - -- **Priority:** P1 (core cognitive capability) -- **Status:** Complete -- **Effort:** 8h design -- **Depends on:** Phase 1 (TokenCounter for L0 budget, DomainEventBus for consolidation), Phase 2 (pipeline integration points) - -Design 3-tier memory system, temporal KG, consolidation pipeline, L0/L1/L2 context tiering, and smart auto-inject retrieval. This phase transforms GoClaw from append-only memory to a curated, searchable, self-maintaining knowledge system. - -## Key Insights - -- Current memory is append-only daily .md files, never revisited. Contradictions accumulate, no consolidation, no dedup -- KG extraction is synchronous (blocks loop). Dedup thresholds hardcoded (0.98/0.90). No temporal awareness -- stale facts never expire -- Memory search is hybrid FTS+vector but flat -- no tier awareness, no relevance-based injection -- Karpathy insight: memory should be curated wiki, not append-only log -- OpenViking insight: L0/L1/L2 tiering saves 10x tokens vs full-context loading -- Graphiti insight: temporal validity windows on KG facts enable contradiction resolution - ---- - -## A. 3-Tier Memory Schema - -### Tier 1: Working Memory - -In context window. No new storage needed. Managed by MessageBuffer in pipeline. - -- Content: recent 20-50 messages (conversation turns) -- Lifecycle: per-turn, free (no persistence cost) -- Size: up to 80% of context window budget -- Pruned by: PruneStage (token-accurate via TokenCounter) - -### Tier 2: Episodic Memory - -New table. Session summaries + key interactions with pgvector embeddings. - -```sql --- Migration: 000XXX_episodic_summaries.up.sql - -CREATE TABLE episodic_summaries ( - id UUID PRIMARY KEY DEFAULT gen_random_uuid(), - tenant_id UUID NOT NULL REFERENCES tenants(id) ON DELETE CASCADE, - agent_id UUID NOT NULL REFERENCES agents(id) ON DELETE CASCADE, - user_id VARCHAR(255) NOT NULL DEFAULT '', - session_key VARCHAR(500) NOT NULL, - - -- Content - summary TEXT NOT NULL, -- LLM-generated session summary - key_entities JSONB DEFAULT '[]', -- entity names extracted (for downstream KG) - key_topics JSONB DEFAULT '[]', -- topic tags for BM25 - message_count INT NOT NULL DEFAULT 0, -- messages in summarized session - token_count INT NOT NULL DEFAULT 0, -- total tokens in session - - -- Embeddings for semantic search - embedding vector(1536), - tsv tsvector GENERATED ALWAYS AS (to_tsvector('simple', summary)) STORED, - - -- L0 abstract (pre-computed, ~50 tokens) - l0_abstract TEXT NOT NULL DEFAULT '', - - -- Lifecycle - source_id VARCHAR(255) NOT NULL DEFAULT '', -- dedup key (session_key + compaction_count) - created_at TIMESTAMPTZ DEFAULT NOW(), - expires_at TIMESTAMPTZ, -- NULL = no expiry, default 90d from creation - - UNIQUE(agent_id, user_id, source_id) -); - -CREATE INDEX idx_episodic_scope ON episodic_summaries(agent_id, user_id); -CREATE INDEX idx_episodic_tenant ON episodic_summaries(tenant_id); -CREATE INDEX idx_episodic_created ON episodic_summaries(agent_id, user_id, created_at DESC); -CREATE INDEX idx_episodic_expires ON episodic_summaries(expires_at) WHERE expires_at IS NOT NULL; -CREATE INDEX idx_episodic_tsv ON episodic_summaries USING gin(tsv); -CREATE INDEX idx_episodic_embedding ON episodic_summaries USING ivfflat (embedding vector_cosine_ops) - WITH (lists = 50); -``` - -### Tier 3: Semantic Memory - -Existing KG tables (`kg_entities`, `kg_relations`) + temporal fields (section B). Facts extracted from episodic summaries via consolidation pipeline. Permanent, deduplicated. - -No new table needed. Temporal fields added to existing tables. - -### Tier Flow - -``` -Working (Tier 1) Episodic (Tier 2) Semantic (Tier 3) -[in context window] [episodic_summaries] [kg_entities + kg_relations] - | | | - |-- session end / compaction ------>| | - | (session.completed event) | | - | |-- extraction worker -------->| - | | (episodic.created event) | - | | | - |<-- auto-inject L0 summaries ------| | - |<-- memory_search L1/L2 ---------->| | - |<-- kg_search ------------------------------------------->| | -``` - ---- - -## B. Temporal KG Schema - -### Schema Changes - -```sql --- Migration: 000XXX_kg_temporal.up.sql - --- Add temporal validity to entities -ALTER TABLE kg_entities - ADD COLUMN valid_from TIMESTAMPTZ DEFAULT NOW(), - ADD COLUMN valid_until TIMESTAMPTZ; -- NULL = currently valid - --- Add temporal validity to relations -ALTER TABLE kg_relations - ADD COLUMN valid_from TIMESTAMPTZ DEFAULT NOW(), - ADD COLUMN valid_until TIMESTAMPTZ; - --- Backfill existing data: valid_from = created_at, valid_until = NULL (current) -UPDATE kg_entities SET valid_from = created_at WHERE valid_from IS NULL; -UPDATE kg_relations SET valid_from = created_at WHERE valid_from IS NULL; - --- Partial index for current facts (most common query) -CREATE INDEX idx_kg_entities_current ON kg_entities(agent_id, user_id) - WHERE valid_until IS NULL; -CREATE INDEX idx_kg_relations_current ON kg_relations(agent_id, user_id) - WHERE valid_until IS NULL; - --- Add configurable dedup thresholds (per-agent, stored in agents.other_config JSONB) --- No schema change needed -- uses existing agents.other_config field: --- { "kg_dedup_auto_threshold": 0.98, "kg_dedup_flag_threshold": 0.90 } -``` - -### Query Patterns - -```sql --- Current facts (default, most queries) -SELECT * FROM kg_entities -WHERE agent_id = $1 AND user_id = $2 AND valid_until IS NULL; - --- Historical: what was true at a specific time? -SELECT * FROM kg_entities -WHERE agent_id = $1 AND user_id = $2 - AND valid_from <= $3 - AND (valid_until IS NULL OR valid_until >= $3); - --- Supersede a fact: new contradicting fact arrives -UPDATE kg_entities SET valid_until = NOW(), updated_at = NOW() -WHERE agent_id = $1 AND user_id = $2 AND external_id = $3 AND valid_until IS NULL; --- Then INSERT new entity with valid_from = NOW() -``` - -### Contradiction Resolution - -When extraction produces a fact that contradicts an existing current fact: - -``` -1. Extractor produces: "User works at CompanyB" (confidence 0.85) -2. Existing current fact: "User works at CompanyA" (confidence 0.90) -3. If same external_id + same entity_type: - a. Supersede old: SET valid_until = NOW() on CompanyA fact - b. Insert new: CompanyB with valid_from = NOW() -4. If different external_id but related via relation: - a. Flag as dedup candidate (human review) -``` - -### Configurable Thresholds - -```go -// Per-agent KG config (stored in agents.other_config JSONB) -type KGConfig struct { - DedupAutoThreshold float64 `json:"kg_dedup_auto_threshold"` // default 0.98 - DedupFlagThreshold float64 `json:"kg_dedup_flag_threshold"` // default 0.90 - ExtractionMinConf float64 `json:"kg_extraction_min_conf"` // default 0.75 - EnableTemporal bool `json:"kg_enable_temporal"` // default true -} -``` - ---- - -## C. Consolidation Pipeline - -Event-driven, async, non-blocking. User never waits for consolidation. - -### Event Flow - -``` -session.completed - | - v -EpisodicWorker - |-- Summarize session messages via LLM - |-- Extract key entities (names only, for downstream) - |-- Generate L0 abstract (~1 sentence, ~50 tokens) - |-- Store in episodic_summaries - |-- Generate embedding (async) - |-- Publish: episodic.created - | - v -SemanticExtractionWorker - |-- Read episodic summary - |-- Run KG extractor on summary (not full session -- cheaper) - |-- Upsert entities + relations with temporal fields - |-- Publish: entity.upserted - | - v -DedupWorker - |-- Read newly upserted entity IDs - |-- Compare embeddings against existing entities - |-- Auto-merge (> auto_threshold + name match) - |-- Flag candidates (> flag_threshold) - |-- Handle temporal supersession for contradictions -``` - -### Worker Implementations - -```go -// Package: internal/consolidation - -// EpisodicWorker processes session.completed events. -type EpisodicWorker struct { - store store.EpisodicStore - provider providers.Provider // for LLM summarization - tokenCounter tokencount.TokenCounter - eventBus eventbus.DomainEventBus -} - -func (w *EpisodicWorker) Handle(ctx context.Context, event eventbus.DomainEvent) error { - payload := event.Payload.(eventbus.SessionCompletedPayload) - - // Skip if already processed (idempotent by source_id) - sourceID := fmt.Sprintf("%s:%d", payload.SessionKey, payload.CompactionCount) - if w.store.ExistsBySourceID(ctx, event.AgentID, event.UserID, sourceID) { - return nil // already processed - } - - // 1. Summarize session - summary, err := w.summarizeSession(ctx, payload) - if err != nil { - return fmt.Errorf("summarize: %w", err) - } - - // 2. Extract key entities (lightweight -- name extraction, not full KG) - entities := extractEntityNames(summary) - - // 3. Generate L0 abstract - l0 := generateL0Abstract(summary) // extractive: first sentence or topic sentence - - // 4. Store - ep := &store.EpisodicSummary{ - AgentID: event.AgentID, - UserID: event.UserID, - TenantID: event.TenantID, - SessionKey: payload.SessionKey, - Summary: summary, - KeyEntities: entities, - L0Abstract: l0, - MessageCount: payload.MessageCount, - TokenCount: payload.TokensUsed, - SourceID: sourceID, - ExpiresAt: timePtr(time.Now().Add(90 * 24 * time.Hour)), // 90d TTL - } - if err := w.store.Create(ctx, ep); err != nil { - return fmt.Errorf("store episodic: %w", err) - } - - // 5. Publish for downstream - w.eventBus.Publish(eventbus.DomainEvent{ - Type: eventbus.EventEpisodicCreated, - AgentID: event.AgentID, - UserID: event.UserID, - Payload: eventbus.EpisodicCreatedPayload{ - EpisodicID: ep.ID, - SessionKey: payload.SessionKey, - Summary: summary, - KeyEntities: entities, - }, - }) - - return nil -} -``` - -### Summarization Prompt (Episodic) - -``` -Summarize this conversation session concisely. Focus on: -- Key decisions made -- Facts learned about the user or project -- Tasks completed or in-progress -- Important technical details -- User preferences expressed - -Output: 2-4 paragraph summary. Include entity names (people, projects, tools) explicitly. -Do NOT include greetings, filler, or metadata. -``` - -### L0 Abstract Generation - -V3.0: extractive (first meaningful sentence of summary). V3.1+: LLM-generated. - -```go -// generateL0Abstract produces a ~50 token abstract from episodic summary. -// V3.0: extractive (first sentence). Future: LLM-generated. -func generateL0Abstract(summary string) string { - // Split into sentences, take first non-trivial one - sentences := splitSentences(summary) - for _, s := range sentences { - s = strings.TrimSpace(s) - if len(s) > 20 { // skip very short fragments - if len(s) > 200 { // truncate if too long - s = s[:200] + "..." - } - return s - } - } - // Fallback: first 200 chars - if len(summary) > 200 { - return summary[:200] + "..." - } - return summary -} -``` - ---- - -## D. L0/L1/L2 Context Tiering - -### Layer Definitions - -| Layer | Content | Token cost | When loaded | Storage | -|-------|---------|-----------|-------------|---------| -| **L0** | 1-sentence abstract | ~50 tokens | Always (auto-inject if relevant) | `episodic_summaries.l0_abstract` | -| **L1** | Structured overview | ~200 tokens | When topic relevant | Generated on-demand, cached | -| **L2** | Full content | Full | On-demand via tool call | `episodic_summaries.summary` (episodic) or original document (memory docs) | - -### L1 Generation - -On-demand when agent requests deeper context or auto-inject detects high relevance. - -```go -// L1Cache caches generated L1 overviews. LRU, 500 entries, 1h TTL. -type L1Cache struct { - cache *lru.Cache[string, string] // key: episodic_id or doc_path -} - -// GenerateL1 creates structured overview from episodic summary or memory doc. -func (c *L1Cache) GenerateL1(id string, fullContent string) string { - if cached, ok := c.cache.Get(id); ok { - return cached - } - // Extractive: key bullet points from full content - l1 := extractKeyPoints(fullContent, 200) // ~200 tokens budget - c.cache.Add(id, l1) - return l1 -} -``` - -### L2 Access - -Full content loaded via tool call. Agent calls `memory_search(query, depth="full")` or `memory_expand(id)`. - -```go -// memory_expand tool: loads L2 full content for a specific memory entry -// Parameters: id (episodic ID or document path) -// Returns: full content (summary or document) -``` - ---- - -## E. Smart Auto-Inject Retrieval - -Per-turn relevance check + L0 injection. Non-blocking, lightweight. - -### Flow - -``` -User message arrives (in ContextStage) - | - v -1. Extract intent keywords from user message - - Tokenize, remove stopwords, keep nouns/verbs/proper nouns - - If message too short ("hello", "ok"): skip injection (no relevant context) - | - v -2. BM25 + embedding relevance check against episodic index - - BM25 on episodic_summaries.tsv (fast, FTS) - - Embedding similarity on episodic_summaries.embedding (if available) - - Merge scores (configurable weights) - | - v -3. Filter by relevance threshold - - Default: 0.3 (tunable per agent via agent config) - - If no results above threshold: skip injection - | - v -4. Inject "## Memory Context" section into system prompt - - Max 5 entries, max ~200 tokens total (L0 summaries only) - - Format per entry: "- {topic}: {L0 abstract} [use memory_search for details]" - | - v -5. Agent sees hint, can call memory_search for L1/L2 -``` - -### Interface - -```go -// Package: internal/memory - -// AutoInjector checks relevance and produces L0 injection for system prompt. -type AutoInjector interface { - // Inject checks user message against memory index. - // Returns formatted section for system prompt, or "" if nothing relevant. - // Budget: max ~200 tokens of L0 summaries. - Inject(ctx context.Context, params InjectParams) (string, error) -} - -type InjectParams struct { - AgentID string - UserID string - TenantID string - UserMessage string - MaxEntries int // default 5 - MaxTokens int // default 200 - Threshold float64 // relevance threshold (default 0.3) -} - -// InjectResult for observability -type InjectResult struct { - Section string // formatted prompt section - MatchCount int // total matches found - Injected int // entries injected (after budget trim) - TopScore float64 // highest relevance score -} -``` - -### System Prompt Section (generated by AutoInjector) - -``` -## Memory Context - -Relevant memories from past sessions (summaries only -- use memory_search for details): -- Project Alpha: Discussed migration from PostgreSQL to CockroachDB, decided to stay with PG for now. -- User preferences: Prefers Go over Python for backend, uses Neovim as primary editor. -- Bug fix: Resolved auth token expiration issue in session middleware (2026-03-15). - -Use `memory_search` tool to retrieve full details on any topic above. -``` - -### Agent Config - -```go -// Per-agent memory config (extends existing config.MemoryConfig) -type MemoryConfig struct { - // Existing fields... - Enabled bool `json:"enabled"` - - // V3 additions - AutoInjectEnabled bool `json:"auto_inject_enabled"` // default true - AutoInjectThreshold float64 `json:"auto_inject_threshold"` // default 0.3 - AutoInjectMaxTokens int `json:"auto_inject_max_tokens"` // default 200 - EpisodicTTLDays int `json:"episodic_ttl_days"` // default 90 (0 = no expiry) - ConsolidationEnabled bool `json:"consolidation_enabled"` // default true -} -``` - ---- - -## F. MemoryStore v3 Interface - -Extends existing `MemoryStore` with tier-aware operations. - -```go -// Package: internal/store - -// EpisodicSummary represents a Tier 2 episodic memory entry. -type EpisodicSummary struct { - ID string `json:"id"` - TenantID string `json:"tenant_id"` - AgentID string `json:"agent_id"` - UserID string `json:"user_id"` - SessionKey string `json:"session_key"` - Summary string `json:"summary"` - KeyEntities []string `json:"key_entities"` - KeyTopics []string `json:"key_topics"` - L0Abstract string `json:"l0_abstract"` - MessageCount int `json:"message_count"` - TokenCount int `json:"token_count"` - SourceID string `json:"source_id"` - CreatedAt time.Time `json:"created_at"` - ExpiresAt *time.Time `json:"expires_at,omitempty"` -} - -// EpisodicSearchResult is a search hit with tier indicator. -type EpisodicSearchResult struct { - EpisodicID string `json:"episodic_id"` - L0Abstract string `json:"l0_abstract"` // always populated - Score float64 `json:"score"` - CreatedAt time.Time `json:"created_at"` - SessionKey string `json:"session_key"` -} - -// EpisodicStore manages Tier 2 episodic memory. -type EpisodicStore interface { - // CRUD - Create(ctx context.Context, ep *EpisodicSummary) error - Get(ctx context.Context, id string) (*EpisodicSummary, error) - Delete(ctx context.Context, id string) error - List(ctx context.Context, agentID, userID string, limit, offset int) ([]EpisodicSummary, error) - - // Search (hybrid FTS + vector, returns L0 by default) - Search(ctx context.Context, query string, agentID, userID string, opts EpisodicSearchOptions) ([]EpisodicSearchResult, error) - - // Lifecycle - ExistsBySourceID(ctx context.Context, agentID, userID, sourceID string) (bool, error) - PruneExpired(ctx context.Context) (int, error) // delete where expires_at < NOW() - - // Embedding - SetEmbeddingProvider(provider EmbeddingProvider) - Close() error -} - -type EpisodicSearchOptions struct { - MaxResults int - MinScore float64 - VectorWeight float64 - TextWeight float64 -} - -// MemorySearchResultV3 extends search result with tier indicator. -type MemorySearchResultV3 struct { - MemorySearchResult // embed existing fields - Tier string `json:"tier"` // "working", "episodic", "semantic" - EpisodicID string `json:"episodic_id,omitempty"` - L0Abstract string `json:"l0_abstract,omitempty"` -} - -// UnifiedMemorySearch combines Tier 2 (episodic) and existing memory doc search. -// Used by memory_search tool. -type UnifiedMemorySearch interface { - // Search across all tiers. Returns results with tier indicators. - Search(ctx context.Context, query string, agentID, userID string, opts UnifiedSearchOptions) ([]MemorySearchResultV3, error) - - // Expand loads full content for a specific memory entry (L2). - Expand(ctx context.Context, id string, tier string) (string, error) -} - -type UnifiedSearchOptions struct { - MaxResults int - MinScore float64 - Tiers []string // filter: ["episodic", "document"] (empty = all) - Depth string // "l0", "l1", "l2" (default "l1") -} -``` - -### KnowledgeGraphStore v3 Extensions - -```go -// Temporal query additions to existing KnowledgeGraphStore interface. - -// TemporalQueryOptions extends entity queries with time awareness. -type TemporalQueryOptions struct { - AsOf *time.Time // point-in-time query (nil = current only) - IncludeExpired bool // include superseded facts -} - -// New methods added to KnowledgeGraphStore: - -// ListEntitiesTemporal queries entities with temporal awareness. -// When opts.AsOf is nil, returns current facts only (valid_until IS NULL). -// When opts.AsOf is set, returns facts valid at that timestamp. -ListEntitiesTemporal(ctx context.Context, agentID, userID string, - listOpts EntityListOptions, temporal TemporalQueryOptions) ([]Entity, error) - -// SupersedeEntity marks entity as no longer valid and inserts replacement. -// Atomic: UPDATE old SET valid_until=NOW() + INSERT new in single tx. -SupersedeEntity(ctx context.Context, old *Entity, replacement *Entity) error - -// GetDedupConfig returns per-agent dedup thresholds from agents.other_config. -// Returns defaults if not configured. -GetDedupConfig(ctx context.Context, agentID string) (*KGConfig, error) -``` - -### Entity Struct v3 - -```go -// Extended Entity with temporal fields. -type Entity struct { - // ... existing fields unchanged ... - ID string `json:"id"` - AgentID string `json:"agent_id"` - UserID string `json:"user_id,omitempty"` - ExternalID string `json:"external_id"` - Name string `json:"name"` - EntityType string `json:"entity_type"` - Description string `json:"description,omitempty"` - Properties map[string]string `json:"properties,omitempty"` - SourceID string `json:"source_id,omitempty"` - Confidence float64 `json:"confidence"` - CreatedAt int64 `json:"created_at"` - UpdatedAt int64 `json:"updated_at"` - - // V3 temporal fields - ValidFrom *time.Time `json:"valid_from,omitempty"` - ValidUntil *time.Time `json:"valid_until,omitempty"` // nil = currently valid -} - -// Relation with temporal fields (same pattern). -type Relation struct { - // ... existing fields unchanged ... - - // V3 temporal fields - ValidFrom *time.Time `json:"valid_from,omitempty"` - ValidUntil *time.Time `json:"valid_until,omitempty"` -} -``` - ---- - -## Pipeline Integration Points - -### Where Memory/KG Connects to Pipeline Stages - -| Pipeline stage | Memory/KG interaction | -|---------------|----------------------| -| **ContextStage** | AutoInjector.Inject() -> produces ## Memory Context section | -| **PruneStage** | Triggers MemoryFlushStage before compaction | -| **MemoryFlushStage** | Writes to memory docs (existing). Publishes `session.completed` | -| **FinalizeStage** | Publishes `session.completed` for episodic consolidation | -| **ToolStage** | `memory_search` tool calls UnifiedMemorySearch | -| **ToolStage** | `memory_expand` tool calls UnifiedMemorySearch.Expand() | -| **ToolStage** | `kg_search` tool uses temporal queries (default: current facts) | - -### Consolidation Pipeline Registration - -```go -// At gateway startup, register consolidation workers with DomainEventBus. - -func RegisterConsolidationPipeline(eb eventbus.DomainEventBus, deps ConsolidationDeps) { - episodicWorker := NewEpisodicWorker(deps) - semanticWorker := NewSemanticExtractionWorker(deps) - dedupWorker := NewDedupWorker(deps) - - eb.Subscribe(eventbus.EventSessionCompleted, episodicWorker.Handle) - eb.Subscribe(eventbus.EventEpisodicCreated, semanticWorker.Handle) - eb.Subscribe(eventbus.EventEntityUpserted, dedupWorker.Handle) -} -``` - ---- - -## Requirements - -### Functional -- F1: Episodic summaries created automatically after session completion -- F2: L0 abstracts pre-computed at episodic creation (~50 tokens each) -- F3: Auto-inject produces relevant L0 summaries for user messages (skip for trivial messages) -- F4: Temporal KG queries default to current facts; historical queries via `AsOf` parameter -- F5: Contradiction resolution: supersede old fact, insert new with valid_from = NOW() -- F6: Dedup thresholds configurable per agent (not hardcoded) -- F7: Consolidation pipeline non-blocking (user never waits) -- F8: `memory_search` tool returns results with tier indicators -- F9: `memory_expand` tool loads L2 full content -- F10: Expired episodic entries pruned automatically (cron or lazy) - -### Non-Functional -- NF1: Auto-inject latency < 50ms p99 (BM25 + embedding lookup) -- NF2: Episodic summarization completes within 30s (LLM call) -- NF3: Consolidation pipeline handles 100 session.completed events/minute -- NF4: pgvector index efficient up to 100K embeddings per agent -- NF5: L0 injection budget: max 200 tokens per turn - -## Related Code Files - -### Files to modify - -| File | Change | -|------|--------| -| `internal/store/knowledge_graph_store.go` | Add temporal fields to Entity/Relation, new temporal query methods | -| `internal/store/pg/knowledge_graph*.go` | Implement temporal queries, supersede logic | -| `internal/store/memory_store.go` | No change (existing store retained for Tier 1/document memory) | -| `internal/knowledgegraph/extractor.go` | Source extraction from episodic summaries (lighter than raw messages) | -| `internal/agent/systemprompt.go` | Add ## Memory Context section from AutoInjector | -| `internal/tools/memory_tools.go` | Update memory_search to use UnifiedMemorySearch, add memory_expand | - -### New files to create - -| File | Purpose | -|------|---------| -| `internal/store/episodic_store.go` | EpisodicStore interface + types | -| `internal/store/pg/episodic_summaries.go` | PG implementation | -| `internal/memory/auto_injector.go` | AutoInjector implementation | -| `internal/memory/l1_cache.go` | L1 on-demand generation + LRU cache | -| `internal/consolidation/episodic_worker.go` | session.completed -> episodic summary | -| `internal/consolidation/semantic_worker.go` | episodic.created -> KG extraction | -| `internal/consolidation/dedup_worker.go` | entity.upserted -> dedup check | -| `internal/consolidation/registration.go` | Worker registration with EventBus | -| `migrations/000XXX_episodic_summaries.up.sql` | Episodic table + indexes | -| `migrations/000XXX_kg_temporal.up.sql` | Temporal fields on KG tables | - -## Design Steps - -1. Define `episodic_summaries` DDL with indexes -2. Define `EpisodicStore` interface -3. Design temporal KG schema changes (ALTER TABLE + backfill) -4. Define temporal query methods for KnowledgeGraphStore -5. Design consolidation pipeline event flow -6. Design EpisodicWorker with summarization prompt + L0 generation -7. Design AutoInjector with BM25 + embedding relevance check -8. Define UnifiedMemorySearch interface for cross-tier search -9. Design L0/L1/L2 depth parameter for search results -10. Define per-agent memory config extensions - -## Todo List - -- [x] A1: episodic_summaries DDL — documented in phase (SQL in Phase 6) -- [x] A2: EpisodicStore interface — `internal/store/episodic_store.go` -- [x] A3: EpisodicSummary struct + EpisodicSearchResult — `internal/store/episodic_store.go` -- [x] B1: kg_entities temporal fields DDL — documented in phase (SQL in Phase 6) -- [x] B2: kg_relations temporal fields DDL — documented in phase (SQL in Phase 6) -- [x] B3: Backfill migration — documented in phase design -- [x] B4: Temporal query methods — `internal/store/knowledge_graph_temporal.go` -- [x] B5: SupersedeEntity atomic method — `internal/store/knowledge_graph_temporal.go` -- [x] B6: Configurable dedup thresholds — `internal/store/knowledge_graph_temporal.go` KGConfig -- [x] C1: Consolidation event flow design — documented in phase -- [x] C2: EpisodicWorker handler — `internal/consolidation/workers.go` (stub) -- [x] C3: SemanticExtractionWorker handler — `internal/consolidation/workers.go` (stub) -- [x] C4: DedupWorker handler — `internal/consolidation/workers.go` (stub) -- [x] C5: Worker registration at startup — `internal/consolidation/workers.go` Register() -- [x] D1: L0 abstract generation — documented in phase design (extractive v3.0) -- [x] D2: L1 on-demand generation + LRU cache — documented in phase design -- [x] D3: L2 access via memory_expand tool — documented in phase design -- [x] E1: AutoInjector interface — `internal/memory/auto_injector.go` -- [x] E2: Relevance check (BM25 + embedding) — documented in phase design -- [x] E3: System prompt ## Memory Context section — documented in phase design -- [x] E4: Per-agent threshold config — `internal/memory/auto_injector.go` MemoryConfig -- [x] F1: UnifiedMemorySearch interface — documented in phase design -- [x] F2: MemorySearchResultV3 with tier indicator — documented in phase design -- [x] F3: memory_search tool update — documented in phase design -- [x] F4: memory_expand tool definition — documented in phase design - -## Success Criteria - -- episodic_summaries DDL passes `psql` validation -- Temporal KG migration backward-compatible (existing data has valid_from=created_at, valid_until=NULL) -- All consolidation workers idempotent (safe to replay events) -- AutoInjector returns "" for trivial messages ("hello", "ok") -- AutoInjector returns relevant L0s for topic-specific messages -- UnifiedMemorySearch merges results across tiers with correct tier labels -- Per-agent dedup thresholds configurable, defaults match current hardcoded values - -## Risk Assessment - -| Risk | Severity | Mitigation | -|------|----------|------------| -| Episodic summarization LLM cost | MEDIUM | Use smaller/cheaper model for summarization (e.g. GPT-4o-mini). Configurable per agent. | -| Auto-inject false positives (irrelevant memories injected) | MEDIUM | Conservative default threshold (0.3). Agent can override. Monitor via evolution metrics. | -| pgvector index performance at scale (100K+ embeddings) | MEDIUM | ivfflat index with lists=50. Benchmark needed. Increase lists for larger datasets. | -| Consolidation pipeline backlog during high load | LOW | Lane-based concurrency in EventBus. Workers are idempotent. Backlog clears naturally. | -| L0 extractive quality insufficient | LOW | Extractive is "good enough" for v3.0. LLM-generated L0 in v3.1. | -| Temporal backfill migration slow on large KG | LOW | Backfill is simple UPDATE SET DEFAULT. Add timeout guard. | -| Cross-agent memory leakage | HIGH | All queries scoped by tenant_id + agent_id + user_id. Verify in code review. | - -## Security Considerations - -- Tenant isolation: ALL episodic + KG queries must include tenant_id scope. Missing tenant_id = data leak. -- User isolation: episodic summaries scoped by user_id unless shared memory enabled -- L0/L1 content is derived from user conversations -- same access controls as original messages -- Consolidation workers run in system context -- must not expose cross-tenant data -- AutoInjector results filtered by same agent_id + user_id scope as memory_search - -## Unresolved Questions - -1. **Episodic TTL**: 90 days default. Should this be configurable per agent? (Likely yes, via MemoryConfig) -2. **L0 for memory documents**: Should existing memory_documents also get L0 abstracts? (Deferred to v3.1 -- episodic first) -3. **Cross-agent memory**: Should agents in same tenant share semantic memory? (Brainstorm Q2, deferred) -4. **Embedding model**: Continue OpenAI text-embedding-3-small or switch to open model? (Brainstorm Q3, deferred) -5. **Concurrent consolidation**: Multiple sessions ending simultaneously -> same episodic worker. EventBus dedup by source_id handles this, but verify ordering. - -## Next Steps - -- Phase 4 (System Integration) integrates AutoInjector into ContextStage template -- Phase 5 (Orchestration) uses self-evolution metrics from consolidation pipeline -- Phase 6 (Migration) handles v2 daily memory files -> episodic import diff --git a/plans/260406-1304-goclaw-v3-core-design/phase-04-system-integration-design.md b/plans/260406-1304-goclaw-v3-core-design/phase-04-system-integration-design.md deleted file mode 100644 index a44a7009..00000000 --- a/plans/260406-1304-goclaw-v3-core-design/phase-04-system-integration-design.md +++ /dev/null @@ -1,707 +0,0 @@ -# Phase 4: System Integration Design - -## Context Links -- Brainstorm: `plans/reports/brainstorm-260406-1059-goclaw-v3-core-redesign.md` (sections 9, 12, 14, 7) -- Architecture report: `plans/reports/Explore-260406-thorough-v3-architecture.md` (sections 5, 6) -- Workspace forcing: `plans/reports/Explore-260406-workspace-forcing.md` -- Current system prompt: `internal/agent/systemprompt.go`, `internal/agent/systemprompt_sections.go` -- Current tool registry: `internal/tools/registry.go`, `internal/tools/policy.go`, `internal/tools/types.go` -- Current skills: `internal/skills/loader.go`, `internal/skills/search.go`, `internal/store/skill_store.go` - -## Overview -- **Priority:** P1 (core dependency for pipeline loop + memory system) -- **Status:** Complete -- **Effort:** 6h design - -Redesigns 4 tightly-coupled systems that sit between the agent loop and the LLM: -1. Template-based system prompt (replace string concatenation) -2. Tool registry with typed capabilities + declarative policy -3. Skills system with embedding search + ACL + dependency graph -4. Retrieval integration connecting auto-inject L0 with template + tools - ---- - -## Key Insights - -1. Current `BuildSystemPrompt()` is ~200 lines of conditional string building. 15+ sections toggled by booleans. Adding workspace enforcement language or memory context requires touching fragile concatenation logic. -2. Tool types use 12+ optional interfaces (`InterceptorAware`, `PathAllowable`, etc.) with type assertions at registration. No capability metadata on tools themselves. -3. PolicyEngine's 7-step pipeline works but all checks are string-based group matching. No typed capabilities (read-only vs mutating), no workspace-awareness. -4. Skills BM25-only search misses intent-based queries ("help me debug" won't find "debugger" skill). `EmbeddingSkillSearcher` interface exists in store but not wired to loader's `BuildSummary()`. -5. Workspace confusion root cause: system prompt text says "use relative paths" but doesn't state the enforcement rule. Template must carry WorkspaceContext enforcement language verbatim. - ---- - -## Requirements - -### Functional -- F1: System prompt built from Go `text/template` with embedded template file -- F2: Each template section independently toggleable via PromptConfig -- F3: WorkspaceContext generates explicit enforcement language (not suggestive) -- F4: Tools declare capabilities (read-only, mutating, async, mcp-bridged) at registration -- F5: Policy engine evaluates capabilities (e.g., "deny all mutating tools for viewer role") -- F6: Tools receive WorkspaceContext, not raw path strings -- F7: Skills searched by both BM25 (lexical) and embedding (semantic), results merged -- F8: Skills filtered by agent/tenant ACL before injection -- F9: Skill dependency graph: loader resolves + validates before activation -- F10: Auto-inject L0 memory summaries into template's Memory section -- F11: Provider-specific template variants (e.g., Codex needs different tool format) - -### Non-Functional -- NF1: Template rendering <1ms (no LLM calls during prompt build) -- NF2: Skill embedding search <50ms at 1000 skills -- NF3: Backward compatible: v2 PromptConfig can produce identical v2 output -- NF4: Template testable in isolation (no DB dependency for rendering) - ---- - -## Architecture - -### A. Template-Based System Prompt - -#### PromptConfig Interface Contract - -```go -// PromptConfig controls which template sections to include. -// Each field maps to a named template block. -type PromptConfig struct { - // Section toggles - Identity bool - Persona bool // SOUL.md content - Instructions bool // AGENTS.md content - Tools bool - Skills bool - Team bool - Workspace bool - Memory bool // auto-inject L0 section - Sandbox bool - Recency bool // recency reminder - - // Data payloads (populated when section enabled) - IdentityData IdentityData - PersonaContent string // raw SOUL.md - InstructionContent string // raw AGENTS.md - ToolsData ToolsSectionData - SkillsData SkillsSectionData - TeamData TeamSectionData - WorkspaceData WorkspaceSectionData - MemoryData MemorySectionData - SandboxData SandboxSectionData - ExtraPrompt string - - // Provider variant (selects template file) - ProviderVariant string // "" = default, "codex", "dashscope" -} -``` - -#### Section Data Types - -```go -type IdentityData struct { - AgentName string - Emoji string - Model string - Channel string - ChatTitle string - PeerKind string // "direct" or "group" -} - -type ToolsSectionData struct { - ToolDefs []ToolSummary // name + one-line description - MCPTools []ToolSummary // MCP bridge tools (separate listing) - CoreSummary map[string]string // builtin tool summaries -} - -type ToolSummary struct { - Name string - Description string - Capability ToolCapability // read-only, mutating, async, mcp-bridged -} - -type SkillsSectionData struct { - Mode string // "inline" (inject summaries) or "search" (search tool only) - Summaries []SkillSummaryEntry -} - -type SkillSummaryEntry struct { - Slug string - Name string - Description string - Score float64 // relevance score from search -} - -type TeamSectionData struct { - TeamWorkspace string - Members []TeamMemberEntry - Guidance string // edition-specific (Full vs Lite) - TeamMDContent string // TEAM.md content -} - -type TeamMemberEntry struct { - AgentKey string - DisplayName string - Skills string - Role string // "lead" or "member" -} - -type WorkspaceSectionData struct { - ActivePath string - Scope string // "personal" | "team-shared" | "team-isolated" | "delegate" - Enforced bool // always true in v3 - ReadOnlyPaths []string - SharedPath *string - ContextFiles []string // list of managed context files - EnforcementMsg string // pre-rendered enforcement text -} - -type MemorySectionData struct { - L0Summaries []L0Summary // auto-injected memory hints - HasSearch bool // memory_search tool available - HasKG bool // kg_search tool available -} - -type L0Summary struct { - Topic string - Summary string // 1-sentence abstract - ID string // for memory_expand(id) deep retrieval -} - -type SandboxSectionData struct { - ContainerDir string - AccessLevel string // "full" or "read-only" -} -``` - -#### Template Structure - -Embedded template file at `internal/agent/prompt_template.go.tmpl`: - -``` -{{- /* System Prompt Template — v3 */ -}} - -{{- if .Identity -}} -{{- template "identity" .IdentityData -}} -{{- end -}} - -{{- if .Persona -}} -# Persona -{{ .PersonaContent }} -{{- end -}} - -{{- if .Instructions -}} -# Instructions -{{ .InstructionContent }} -{{- end -}} - -{{- if .Workspace -}} -{{- template "workspace" .WorkspaceData -}} -{{- end -}} - -{{- if .Tools -}} -{{- template "tools" .ToolsData -}} -{{- end -}} - -{{- if .Skills -}} -{{- template "skills" .SkillsData -}} -{{- end -}} - -{{- if .Memory -}} -{{- template "memory" .MemoryData -}} -{{- end -}} - -{{- if .Sandbox -}} -{{- template "sandbox" .SandboxData -}} -{{- end -}} - -{{- if .Team -}} -{{- template "team" .TeamData -}} -{{- end -}} - -{{- if .ExtraPrompt -}} -# Additional Context -{{ .ExtraPrompt }} -{{- end -}} -``` - -#### Workspace Sub-Template (Critical) - -``` -{{- define "workspace" -}} -# Workspace -- **Path:** {{ .ActivePath }} -- **Scope:** {{ .Scope }} -- **ENFORCED:** All file tool paths resolve relative to this directory. Absolute paths outside workspace are REJECTED. -{{- if .ReadOnlyPaths }} -- **Read-only access:** {{ range .ReadOnlyPaths }}{{ . }}, {{ end }} -{{- end }} -{{- if .SharedPath }} -- **Shared area:** {{ .SharedPath }} (read/write by both you and delegator) -{{- end }} -{{- if .ContextFiles }} -- **Managed files:** {{ range .ContextFiles }}{{ . }}, {{ end }}(system-managed, read/write intercepted to DB) -{{- end }} -{{- end -}} -``` - -#### Memory Sub-Template - -``` -{{- define "memory" -}} -# Memory Context -{{- if .L0Summaries }} -{{- range .L0Summaries }} -- **{{ .Topic }}:** {{ .Summary }} [use memory_search("{{ .ID }}") for details] -{{- end }} -{{- end }} -{{- if .HasSearch }} -Use `memory_search` for detailed retrieval. Use `kg_search` for relationship queries. -{{- end }} -{{- end -}} -``` - -#### PromptBuilder Interface - -```go -// PromptBuilder renders system prompts from PromptConfig. -type PromptBuilder interface { - Build(cfg PromptConfig) (string, error) -} - -// TemplatePromptBuilder implements PromptBuilder using text/template. -type TemplatePromptBuilder struct { - templates map[string]*template.Template // variant -> parsed template -} - -// NewTemplatePromptBuilder loads embedded templates. -func NewTemplatePromptBuilder() (*TemplatePromptBuilder, error) - -// Build renders prompt. Selects template variant from cfg.ProviderVariant. -// Returns rendered string. Error only on template parse failure (should not happen with embedded). -func (b *TemplatePromptBuilder) Build(cfg PromptConfig) (string, error) -``` - -#### Provider Variants - -- **Default template:** Standard Anthropic/OpenAI format -- **Codex variant:** Tool descriptions formatted as docstrings (Codex prefers this) -- **DashScope variant:** No streaming+tools section (DashScope limitation) -- Variant selected by `cfg.ProviderVariant` which maps to provider capabilities - ---- - -### B. Tool Registry Redesign - -#### Tool Capability Types - -```go -// ToolCapability describes what a tool can do. -type ToolCapability string - -const ( - CapReadOnly ToolCapability = "read-only" // no side effects (read_file, list_files, memory_search) - CapMutating ToolCapability = "mutating" // modifies state (write_file, exec, edit) - CapAsync ToolCapability = "async" // returns immediately, result via callback - CapMCPBridged ToolCapability = "mcp-bridged" // proxied to external MCP server -) - -// ToolMetadata describes a tool's capabilities and requirements. -type ToolMetadata struct { - Name string - Capabilities []ToolCapability - Group string // "fs", "web", "runtime", "memory", "team", etc. - RequiresWorkspace bool // tool needs WorkspaceContext - ProviderHints map[string]any // provider-specific hints (e.g., cache_control) -} -``` - -#### Enhanced Tool Interface - -```go -// Tool is the base interface all tools must implement. -// V3 adds Metadata() for capability declaration. -type Tool interface { - Name() string - Description() string - Parameters() map[string]any - Execute(ctx context.Context, args map[string]any) *Result - Metadata() ToolMetadata // NEW: declarative capability info -} -``` - -> **Migration note:** Existing tools that don't implement `Metadata()` get a wrapper -> that returns default metadata (capabilities inferred from tool group membership). - -#### WorkspaceContext in Tool Context - -```go -// Tools access WorkspaceContext via context, not struct fields. -// Replaces: ToolWorkspaceFromCtx(), ToolTeamWorkspaceFromCtx() - -func WorkspaceContextFromCtx(ctx context.Context) *WorkspaceContext -func WithWorkspaceContext(ctx context.Context, wc *WorkspaceContext) context.Context -``` - -All file tools use `WorkspaceContextFromCtx(ctx)` instead of separate workspace/team-workspace context keys. WorkspaceContext carries ActivePath, ReadOnlyPaths, SharedPath — one struct, one lookup. - -#### Enhanced PolicyEngine - -```go -// PolicyRule is a declarative access rule. -type PolicyRule struct { - // Match conditions (all must match) - Capabilities []ToolCapability // match tools with these capabilities - Groups []string // match tools in these groups - Names []string // match specific tool names - - // Action - Action PolicyAction // allow or deny - Reason string // audit log reason -} - -type PolicyAction string -const ( - PolicyAllow PolicyAction = "allow" - PolicyDeny PolicyAction = "deny" -) - -// PolicyEngine evaluates tool access via ordered rule list. -// V3 adds capability-based rules alongside existing group/name rules. -type PolicyEngine struct { - globalPolicy *config.ToolsConfig - rules []PolicyRule // declarative rules (evaluated in order, first match wins) -} - -// FilterTools returns allowed tools. Now evaluates both legacy 7-step pipeline -// AND declarative rules. Declarative rules take precedence. -func (pe *PolicyEngine) FilterTools( - registry ToolExecutor, - agentID string, - providerName string, - agentToolPolicy *config.ToolPolicySpec, - groupToolAllow []string, - isSubagent bool, - isLeafAgent bool, - role string, // NEW: RBAC role ("admin", "operator", "viewer") -) []providers.ToolDefinition -``` - -#### Tool Definition Generation for Providers - -```go -// ToolDefAdapter generates provider-specific tool definitions. -// Replaces direct ToProviderDef() calls. -type ToolDefAdapter interface { - // ToProviderDefs converts internal tool list to provider wire format. - // Handles provider-specific quirks (DashScope tool format, Codex docstrings, etc.) - ToProviderDefs(tools []Tool, caps ProviderCapabilities) []providers.ToolDefinition -} -``` - -#### Pipeline Integration - -ToolStage in pipeline receives registry + policy engine: -```go -type ToolStage struct { - registry *Registry - policyEngine *PolicyEngine - adapter ToolDefAdapter -} - -// Execute runs tool, records metrics, returns result for ObserveStage. -func (s *ToolStage) Execute(ctx context.Context, state *RunState) error -``` - -ObserveStage processes tool results: -```go -type ObserveStage struct { - loopDetector *LoopDetector - metricsStore EvolutionMetricsStore // record tool effectiveness -} - -// Execute checks for tool loops, records metrics, decides continue/stop. -func (s *ObserveStage) Execute(ctx context.Context, state *RunState) error -``` - ---- - -### C. Skills System Upgrade - -#### Hybrid Search Interface - -```go -// SkillSearcher provides both lexical and semantic skill search. -type SkillSearcher interface { - // Search returns skills matching query, merging BM25 + embedding results. - // Returns at most limit results, deduplicated by slug. - Search(ctx context.Context, query string, limit int) ([]SkillSearchResult, error) - - // SearchForAgent filters by ACL before search. - SearchForAgent(ctx context.Context, agentID uuid.UUID, userID string, query string, limit int) ([]SkillSearchResult, error) -} - -// HybridSkillSearcher implements SkillSearcher with BM25 + embedding fusion. -type HybridSkillSearcher struct { - bm25Index *skills.Index // existing BM25 index - embSearcher store.EmbeddingSkillSearcher // existing PG embedding search - accessStore store.SkillAccessStore // ACL filtering - bm25Weight float64 // default 0.4 - embedWeight float64 // default 0.6 -} -``` - -#### Score Fusion Strategy - -``` -1. BM25 search → top 2*limit results, normalize scores to [0,1] -2. Embedding search → top 2*limit results, normalize scores to [0,1] -3. Merge by slug: combined_score = bm25Weight * bm25_score + embedWeight * embed_score -4. Sort by combined_score descending -5. Return top limit -``` - -#### ACL Model - -```go -// SkillVisibility controls who can see/use a skill. -type SkillVisibility string - -const ( - SkillVisibilityPublic SkillVisibility = "public" // all agents in tenant - SkillVisibilityPrivate SkillVisibility = "private" // only granted agents - SkillVisibilitySystem SkillVisibility = "system" // system-managed, immutable -) - -// ACL check flow: -// 1. If skill.Visibility == "public" → allow -// 2. If skill.Visibility == "private" → check skill_agent_grants table -// 3. If skill.Visibility == "system" → allow (system skills always accessible) -// Existing SkillAccessStore.ListAccessible() already implements this. -``` - -No new DB tables needed. Existing `managed_skills.visibility` + `skill_agent_grants` + `SkillAccessStore` interface covers ACL. The change is wiring ACL into the search path (currently BM25 search ignores ACL). - -#### Dependency Graph - -```go -// SkillDependency represents skill A requires skill B. -// Stored in skill frontmatter: `depends: [slug-b, slug-c]` -type SkillDependency struct { - Slug string // this skill - Requires []string // slugs of required skills -} - -// DepChecker validates skill dependencies are satisfied. -// Existing internal/skills/dep_checker.go already handles this. -// V3 enhancement: block activation of skill with unresolved deps. -type DepChecker interface { - // Check returns missing dependencies for a skill. - Check(slug string, available []string) []string - - // ResolveOrder returns activation order respecting dependency graph. - // Error if circular dependency detected. - ResolveOrder(slugs []string) ([]string, error) -} -``` - -No new DB schema. Dependencies stored in SKILL.md frontmatter `depends:` field. `dep_checker.go` already parses this. V3 adds `ResolveOrder()` for topological sort + circular detection. - -#### Hot-Reload Enhancement - -```go -// SkillLoader manages skill discovery, loading, and hot-reload. -// V3 adds ACL-aware loading + hybrid search integration. -type SkillLoader interface { - // LoadAll discovers and loads all skills from all tiers. - LoadAll(ctx context.Context) error - - // BuildSummary returns skill summary for system prompt injection. - // V3: uses HybridSkillSearcher instead of BM25-only. - BuildSummary(ctx context.Context, agentID uuid.UUID, userID string, query string) string - - // Version returns current reload version (atomic counter). - Version() int64 - - // OnReload registers callback for hot-reload events. - OnReload(fn func()) -} -``` - ---- - -### D. Retrieval Integration - -#### Auto-Inject L0 Flow - -``` -Per turn (in ContextStage): -1. Extract keywords from latest user message (simple tokenization, same as BM25) -2. Check relevance against memory index: - a. BM25 score against episodic summaries - b. Embedding similarity against semantic memory (KG entities) -3. If max_score > agent.retrieval_threshold (default 0.3): - → Build L0Summaries for PromptConfig.MemoryData - → Cap at max_l0_tokens (default 200 tokens, counted via TokenCounter) -4. PromptBuilder renders Memory section with L0 hints -5. Agent sees hints, can call memory_search(id) for L1/L2 detail -``` - -#### Retrieval Config - -```go -// RetrievalConfig controls auto-inject behavior. Per-agent configurable. -type RetrievalConfig struct { - Enabled bool `json:"enabled"` // default true - RelevanceThreshold float64 `json:"relevance_threshold"` // default 0.3 - MaxL0Tokens int `json:"max_l0_tokens"` // default 200 - MaxL0Items int `json:"max_l0_items"` // default 5 - BM25Weight float64 `json:"bm25_weight"` // default 0.4 - EmbeddingWeight float64 `json:"embedding_weight"` // default 0.6 -} -``` - -Stored in agent settings JSON (`agents.settings` column, existing JSONB field). No new DB columns. - -#### Retrieval Interface - -```go -// Retriever performs auto-inject L0 retrieval for ContextStage. -type Retriever interface { - // RetrieveL0 returns L0 summaries relevant to the query. - // Called once per turn in ContextStage. - RetrieveL0(ctx context.Context, agentID uuid.UUID, userID string, query string, cfg RetrievalConfig) ([]L0Summary, error) -} - -// HybridRetriever implements Retriever using memory + KG stores. -type HybridRetriever struct { - memoryStore store.MemoryStore - kgStore store.KnowledgeGraphStore - tokenCounter TokenCounter - embProvider store.EmbeddingProvider -} -``` - -#### memory_search / kg_search Tool Updates - -```go -// memory_search tool gains L1/L2 mode parameter. -// Args: { "query": "...", "depth": "l1"|"l2" } -// L1: returns structured overview (episodic summaries, ~200 tokens each) -// L2: returns full detail (original content, KG traversal) -// Default: "l1" (backward compatible — current behavior is L1-equivalent) - -// kg_search tool gains temporal parameter. -// Args: { "query": "...", "as_of": "2026-01-01" } -// as_of: returns facts valid at that date. Omit = current facts only. -``` - ---- - -## Related Code Files - -### Files to Modify -| File | Change | -|------|--------| -| `internal/agent/systemprompt.go` | Replace BuildSystemPrompt() with PromptBuilder.Build() | -| `internal/agent/systemprompt_sections.go` | Extract section data builders, remove string concat | -| `internal/agent/loop_history.go` | Use PromptBuilder in buildMessages() | -| `internal/tools/types.go` | Add Metadata() to Tool interface | -| `internal/tools/registry.go` | Store ToolMetadata, add capability queries | -| `internal/tools/policy.go` | Add capability-based rules, role parameter | -| `internal/tools/context_keys.go` | Replace dual workspace keys with WorkspaceContext | -| `internal/tools/filesystem.go` | Use WorkspaceContextFromCtx() | -| `internal/tools/filesystem_write.go` | Use WorkspaceContextFromCtx() | -| `internal/tools/filesystem_list.go` | Use WorkspaceContextFromCtx() | -| `internal/tools/shell.go` | Use WorkspaceContextFromCtx() | -| `internal/skills/loader.go` | Wire HybridSkillSearcher into BuildSummary() | -| `internal/skills/search.go` | Keep BM25, add fusion logic | -| `internal/tools/memory_search.go` | Add depth parameter (l1/l2) | -| `internal/tools/knowledge_graph_search.go` | Add as_of temporal parameter | - -### Files to Create -| File | Purpose | -|------|---------| -| `internal/agent/prompt_builder.go` | TemplatePromptBuilder implementation | -| `internal/agent/prompt_template.go.tmpl` | Embedded template file (default variant) | -| `internal/agent/prompt_template_codex.go.tmpl` | Codex variant | -| `internal/agent/prompt_config_types.go` | PromptConfig, section data types | -| `internal/tools/capability.go` | ToolCapability, ToolMetadata types | -| `internal/tools/workspace_context.go` | WorkspaceContext struct + context helpers | -| `internal/tools/tool_def_adapter.go` | ToolDefAdapter interface + default impl | -| `internal/skills/hybrid_searcher.go` | HybridSkillSearcher | -| `internal/agent/retriever.go` | Retriever interface + HybridRetriever | - ---- - -## Design Steps - -1. Define PromptConfig + all section data types in `prompt_config_types.go` -2. Write default template file with all section sub-templates -3. Implement TemplatePromptBuilder with variant loading -4. Define ToolCapability + ToolMetadata types -5. Add Metadata() to Tool interface with backward-compat wrapper -6. Define WorkspaceContext struct with context helpers -7. Update PolicyEngine: add capability rules + role parameter -8. Define ToolDefAdapter for provider-specific tool def generation -9. Implement HybridSkillSearcher with score fusion -10. Wire ACL into skill search path -11. Define Retriever interface + RetrievalConfig -12. Implement HybridRetriever for L0 auto-inject -13. Update memory_search tool with depth parameter design -14. Update kg_search tool with as_of temporal parameter design -15. Write provider variant templates (Codex, DashScope) - ---- - -## Todo List - -- [x] A1: PromptConfig + section data types — `internal/agent/prompt_config_types.go` -- [x] A2: Default template file — documented in phase design -- [x] A3: TemplatePromptBuilder interface — `internal/agent/prompt_config_types.go` -- [x] A4: Provider variant templates — documented in phase design -- [x] A5: Workspace sub-template — documented in phase design -- [x] B1: ToolCapability + ToolMetadata — `internal/tools/capability.go` -- [x] B2: Backward-compat wrapper — documented in phase design -- [x] B3: WorkspaceContext + context key — done in Phase 1 (`internal/workspace/`) -- [x] B4: PolicyEngine capability rules — documented in phase design -- [x] B5: ToolDefAdapter interface — documented in phase design -- [x] C1: HybridSkillSearcher — documented in phase design -- [x] C2: ACL wiring into search path — documented in phase design -- [x] C3: DepChecker.ResolveOrder() — documented in phase design -- [x] D1: Retriever + RetrievalConfig — `internal/agent/retriever.go` -- [x] D2: L0 auto-inject flow — documented in phase design -- [x] D3: memory_search depth param — documented in phase design -- [x] D4: kg_search as_of temporal param — documented in phase design - ---- - -## Success Criteria - -1. PromptConfig can reproduce exact v2 system prompt output (regression test) -2. All template sections independently toggleable + testable -3. WorkspaceContext carries all info needed by file tools (no separate context keys) -4. Every tool declares capabilities via Metadata() -5. PolicyEngine can express "deny mutating tools for viewer role" in one rule -6. HybridSkillSearcher returns better results than BM25-only on intent queries -7. L0 auto-inject adds <200 tokens per turn, only when relevant -8. Template renders in <1ms (no LLM calls) - ---- - -## Risk Assessment - -| Risk | Severity | Mitigation | -|------|----------|------------| -| Template engine too rigid for edge cases | MEDIUM | Keep ExtraPrompt escape hatch. Provider variants handle outliers. | -| Metadata() breaks all existing tool implementations | HIGH | Backward-compat wrapper: `defaultMetadata(tool)` infers from group/name. Phase implementation. | -| Hybrid skill search slower than BM25-only | LOW | BM25 <1ms, embedding <50ms. Parallel execution. Cache embeddings. | -| WorkspaceContext migration touches every file tool | HIGH | Single context key replaces 2. Grep for `ToolWorkspaceFromCtx` — 15 call sites. Mechanical refactor. | -| L0 auto-inject adds latency to every turn | MEDIUM | Memory index query <10ms. Skip if no memory (hasMemory flag). Threshold prevents injection for low-relevance messages. | -| Provider template variants diverge over time | LOW | Variants inherit from default, override only specific blocks. Test all variants in CI. | - ---- - -## Security Considerations - -- **Template injection:** PromptConfig data (PersonaContent, InstructionContent) comes from DB. Template uses `{{ }}` not `{{ | html }}`. Since output goes to LLM (not browser), HTML escaping not needed. But ensure no Go template directives in user content — use `{{ printf "%s" .Content }}` for raw pass-through. -- **Workspace enforcement:** WorkspaceSectionData.EnforcementMsg must be generated server-side, not from user input. Template renders it verbatim. -- **ACL bypass:** HybridSkillSearcher MUST filter AFTER search, not before (embedding search doesn't support WHERE clauses). Verify: `ListAccessible()` called on search results. -- **Tool capability lying:** Tools self-declare capabilities. PolicyEngine should cross-check: tools in "fs" group claiming "read-only" but actually writing. Add assertion in tests. diff --git a/plans/260406-1304-goclaw-v3-core-design/phase-05-orchestration-intelligence-design.md b/plans/260406-1304-goclaw-v3-core-design/phase-05-orchestration-intelligence-design.md deleted file mode 100644 index 90633482..00000000 --- a/plans/260406-1304-goclaw-v3-core-design/phase-05-orchestration-intelligence-design.md +++ /dev/null @@ -1,923 +0,0 @@ -# Phase 5: Orchestration & Intelligence Design - -## Context Links -- Brainstorm: `plans/reports/brainstorm-260406-1059-goclaw-v3-core-redesign.md` (sections 11, 15) -- Architecture report: `plans/reports/Explore-260406-thorough-v3-architecture.md` (section 8) -- Agent link store: `internal/store/agent_link_store.go` -- Current subagent: `internal/tools/subagent.go`, `internal/tools/subagent_exec.go` -- Current team tools: `internal/tools/team_tool.go`, `internal/tools/team_tool_backend.go` -- Event bus: `internal/bus/bus.go`, `internal/bus/types.go` -- Phase 4: `phase-04-system-integration-design.md` (WorkspaceContext, ToolCapability, PolicyEngine) - -## Overview -- **Priority:** P1 (builds on Phase 4 integration layer) -- **Status:** Complete -- **Effort:** 6h design - -Designs 4 interconnected systems: -1. **Delegate tool** — lightweight inter-agent delegation via agent_links -2. **Orchestration mode config** — layered: spawn always, delegate base, team optional -3. **Self-evolution** — progressive 3-stage rollout (metrics -> suggestions -> auto-adapt) -4. **EventBus integration** — orchestration + evolution events - ---- - -## Key Insights - -1. Agent Links infrastructure exists (`agent_link_store.go`) with `CanDelegate()`, `GetLinkBetween()`, `DelegateTargets()`, even `SearchDelegateTargetsByEmbedding()` — but NO tool uses it. Full plumbing ready, needs ~100 LOC tool wrapper. -2. Current orchestration: spawn (self-clone, ~300 LOC) + team_tasks (9,400 LOC, 18 actions). No middle ground. Simple delegation requires team overhead or spawn limitations. -3. Self-evolution is greenfield. No metrics infrastructure exists. Design must be additive — no existing behavior changes in Stage 1. -4. EventBus (`bus.MessageBus`) is channel-oriented (Inbound/Outbound messages). Broadcast events are typed but payloads are `any`. V3 needs structured event types for orchestration lifecycle. - ---- - -## Requirements - -### Functional -- F1: `delegate` tool — agent delegates task to linked agent, async or sync -- F2: Delegate permission checked via existing `agentLinkStore.CanDelegate()` -- F3: Delegatee uses own workspace + optional shared area + read-only exports -- F4: Orchestration modes configurable per tenant (default) + per agent (override) -- F5: System prompt only shows tools for active orchestration mode -- F6: Self-evolution Stage 1: record metrics (retrieval quality, tool effectiveness, response feedback) -- F7: Self-evolution Stage 2: analyze metrics, generate suggestions, admin review UI -- F8: Self-evolution Stage 3: auto-adapt with guardrails + rollback -- F9: All orchestration lifecycle events published via EventBus - -### Non-Functional -- NF1: Delegate tool adds <5ms overhead over direct agent invocation -- NF2: Metrics recording non-blocking (goroutine, never delays response) -- NF3: Self-evolution stages independently toggleable (feature flags) -- NF4: Backward compatible: existing spawn + team_tasks unaffected - ---- - -## Architecture - -### A. Delegate Tool - -#### Tool Interface - -```go -// DelegateTool enables inter-agent task delegation via agent_links. -// Parameters: -// agent_key (required): target agent key (from known delegate targets) -// task (required): task description for the delegatee -// mode (optional): "async" (default) or "sync" -// context (optional): additional context string passed to delegatee -// files (optional): list of file paths to export (read-only for delegatee) - -type DelegateTool struct { - agentLinkStore store.AgentLinkStore - agentStore store.AgentStore - runner AgentRunner // interface to invoke agent loop - bus bus.EventPublisher -} - -func (t *DelegateTool) Name() string { return "delegate" } - -func (t *DelegateTool) Metadata() ToolMetadata { - return ToolMetadata{ - Name: "delegate", - Capabilities: []ToolCapability{CapAsync}, - Group: "orchestration", - } -} - -func (t *DelegateTool) Parameters() map[string]any { - return map[string]any{ - "type": "object", - "properties": map[string]any{ - "agent_key": map[string]any{ - "type": "string", - "description": "Key of the agent to delegate to (from your known delegate targets)", - }, - "task": map[string]any{ - "type": "string", - "description": "Task description for the delegatee", - }, - "mode": map[string]any{ - "type": "string", - "enum": []string{"async", "sync"}, - "default": "async", - "description": "async: fire-and-forget with result notification. sync: wait for completion.", - }, - "context": map[string]any{ - "type": "string", - "description": "Additional context passed to delegatee's system prompt", - }, - "files": map[string]any{ - "type": "array", - "items": map[string]any{"type": "string"}, - "description": "File paths (relative to your workspace) to share read-only with delegatee", - }, - }, - "required": []string{"agent_key", "task"}, - } -} -``` - -#### Execution Flow - -``` -delegate(agent_key="analyst", task="analyze Q1 data", mode="async") - | - v -1. Resolve target: agentStore.GetByKey(agent_key) - -> fail if agent not found or inactive - | - v -2. Permission check: agentLinkStore.CanDelegate(fromAgentID, toAgentID) - -> fail if no active link - | - v -3. Get link config: agentLinkStore.GetLinkBetween(from, to) - -> read Settings JSON for workspace config - | - v -4. Build WorkspaceContext for delegatee: - - ActivePath: delegatee's own workspace (resolved normally) - - SharedPath: /data/delegates/{linkID}/shared (created if not exists) - - ReadOnlyPaths: exported files from delegator (validated against workspace) - - Scope: "delegate" - | - v -5. Build delegation request: - - ExtraPrompt: "Delegated task from {source_agent_key}: {task}\n{context}" - - WorkspaceContext: as built above - - Mode: PromptMinimal (subagent-style, reduced sections) - | - v -6. Execute: - async mode: - - Spawn goroutine via scheduler lane - - Return immediately: "Task delegated to {agent_key}. You'll be notified on completion." - - On completion: bus.Broadcast(delegate.completed event) - - Result delivered via message bus (same as spawn announce) - sync mode: - - Block until delegatee completes (with timeout from link.Settings) - - Return delegatee's final response directly - | - v -7. Emit event: bus.Broadcast("delegate.sent", DelegateSentPayload{...}) -``` - -#### Workspace Model for Delegation - -```go -// DelegateWorkspaceConfig is stored in agent_link.settings JSON. -type DelegateWorkspaceConfig struct { - // SharedPath: shared directory both agents can read/write. - // Relative to data dir. Default: "delegates/{linkID}/shared" - SharedPath string `json:"shared_path,omitempty"` - - // SourceExports: glob patterns of delegator files shared read-only. - // Relative to delegator's workspace. E.g., ["docs/", "data/results/"] - SourceExports []string `json:"source_exports,omitempty"` - - // SyncTimeout: max wait time for sync mode (seconds). Default: 300 - SyncTimeout int `json:"sync_timeout,omitempty"` - - // MaxConcurrent: max concurrent delegations on this link. 0 = unlimited. - // Already exists as AgentLinkData.MaxConcurrent field. -} -``` - -No new DB tables. Config stored in existing `agent_links.settings` JSONB column. - -#### AgentRunner Interface - -```go -// AgentRunner abstracts invoking an agent's loop for delegation. -// Implemented by the gateway's run dispatcher. -type AgentRunner interface { - // RunDelegation executes a delegation task and returns the result. - // ctx carries workspace context, tenant scope, timeout. - RunDelegation(ctx context.Context, req DelegationRequest) (*DelegationResult, error) -} - -type DelegationRequest struct { - TargetAgentID uuid.UUID - SourceAgentID uuid.UUID - LinkID uuid.UUID - Task string - Context string - WorkspaceCtx *WorkspaceContext - UserID string // inherit from delegator's user context - TenantID uuid.UUID - SyncTimeout time.Duration // 0 = async -} - -type DelegationResult struct { - Response string // delegatee's final text response - ToolCalls int // number of tool calls made - Duration time.Duration - Error error -} -``` - ---- - -### B. Orchestration Mode Configuration - -#### Mode Definitions - -```go -// OrchestrationMode controls which inter-agent tools are available. -type OrchestrationMode string - -const ( - // ModeSpawn: self-clone only. spawn tool available. - // Simplest mode. Agent can only create copies of itself for background work. - ModeSpawn OrchestrationMode = "spawn" - - // ModeDelegate: agent links + spawn. delegate tool available. - // Agent can send tasks to linked agents. Base inter-agent mode. - ModeDelegate OrchestrationMode = "delegate" - - // ModeTeam: full team tasks + delegate + spawn. team_tasks tool available. - // Full lifecycle: review, approval, progress tracking. - // Only available when agent is member of an active team. - ModeTeam OrchestrationMode = "team" -) -``` - -#### Configuration Layering - -```go -// OrchestrationConfig controls orchestration behavior. -type OrchestrationConfig struct { - // Mode: active orchestration mode. - // Tenant sets default, agent overrides. - Mode OrchestrationMode `json:"orchestration_mode"` -} -``` - -**Storage:** No new DB columns. Uses existing config mechanisms: -- Tenant default: `tenant_settings.orchestration_mode` (existing JSONB settings column on tenants table) -- Agent override: `agents.settings -> 'orchestration_mode'` (existing JSONB settings column) - -**Resolution order:** -``` -1. Agent settings.orchestration_mode (if set) -2. Tenant settings.orchestration_mode (if set) -3. Default: "spawn" -``` - -#### Tool Visibility by Mode - -| Mode | spawn | delegate | team_tasks | -|------|-------|----------|------------| -| spawn | yes | no | no | -| delegate | yes | yes | no | -| team | yes | yes | yes | - -**Implementation:** PolicyEngine checks orchestration mode during FilterTools: - -```go -// In PolicyEngine.FilterTools(), after existing 7-step pipeline: -func (pe *PolicyEngine) applyOrchestrationMode( - allowed []string, - mode OrchestrationMode, - isTeamMember bool, -) []string { - switch mode { - case ModeSpawn: - allowed = subtractSet(allowed, []string{"delegate", "team_tasks"}) - case ModeDelegate: - allowed = subtractSet(allowed, []string{"team_tasks"}) - case ModeTeam: - if !isTeamMember { - // Downgrade to delegate if not in team - allowed = subtractSet(allowed, []string{"team_tasks"}) - } - } - return allowed -} -``` - -#### System Prompt Integration - -PromptConfig (from Phase 4) carries orchestration info: - -```go -// Added to PromptConfig: -type OrchestrationSectionData struct { - Mode OrchestrationMode - DelegateTargets []DelegateTargetEntry // from agentLinkStore.DelegateTargets() - TeamContext *TeamSectionData // only if ModeTeam + team member -} - -type DelegateTargetEntry struct { - AgentKey string - DisplayName string - Description string -} -``` - -Template renders delegate targets when mode is delegate/team: - -``` -{{- define "orchestration" -}} -{{- if eq .Mode "delegate" "team" }} -# Delegation -You can delegate tasks to these agents: -{{- range .DelegateTargets }} -- **{{ .AgentKey }}**: {{ .Description }} -{{- end }} -Use `delegate(agent_key, task)` to send work. -{{- end }} -{{- end -}} -``` - ---- - -### C. Self-Evolution Design (Progressive) - -#### Stage 1: Metrics Infrastructure (v3.0) - -##### DB Schema - -```sql --- New table: agent_evolution_metrics -CREATE TABLE agent_evolution_metrics ( - id UUID PRIMARY KEY DEFAULT gen_random_uuid(), - tenant_id UUID NOT NULL REFERENCES tenants(id), - agent_id UUID NOT NULL REFERENCES agents(id) ON DELETE CASCADE, - session_key TEXT NOT NULL, - - metric_type TEXT NOT NULL, -- 'retrieval', 'tool', 'feedback' - metric_key TEXT NOT NULL, -- specific metric (e.g., tool name, retrieval query) - value JSONB NOT NULL, -- metric-specific payload - - created_at TIMESTAMPTZ NOT NULL DEFAULT NOW() -); - --- Indexes for efficient querying -CREATE INDEX idx_evo_metrics_agent_type ON agent_evolution_metrics(agent_id, metric_type); -CREATE INDEX idx_evo_metrics_created ON agent_evolution_metrics(created_at); -CREATE INDEX idx_evo_metrics_tenant ON agent_evolution_metrics(tenant_id); - --- Partition-ready: created_at is in schema for future range partitioning --- For now, TTL cleanup via cron: DELETE WHERE created_at < NOW() - INTERVAL '90 days' -``` - -##### Metric Types - -```go -// MetricType constants -const ( - MetricRetrieval MetricType = "retrieval" - MetricTool MetricType = "tool" - MetricFeedback MetricType = "feedback" -) - -// RetrievalMetricValue records per-retrieval quality. -type RetrievalMetricValue struct { - Query string `json:"query"` - ResultCount int `json:"result_count"` - TopScore float64 `json:"top_score"` - UsedInReply bool `json:"used_in_reply"` // heuristic: result text appears in LLM output - Source string `json:"source"` // "memory_search", "kg_search", "auto_inject" -} - -// ToolMetricValue records per-tool-call effectiveness. -type ToolMetricValue struct { - ToolName string `json:"tool_name"` - Success bool `json:"success"` // !result.IsError - Duration time.Duration `json:"duration_ms"` - ResultSize int `json:"result_size"` // bytes - Capability string `json:"capability"` // from ToolMetadata -} - -// FeedbackMetricValue records implicit satisfaction signal. -type FeedbackMetricValue struct { - Signal string `json:"signal"` // "correction", "followup", "thanks", "new_topic" - TurnGap int `json:"turn_gap"` // turns between response and signal - PreviousTool string `json:"previous_tool"` // last tool used before signal -} -``` - -##### Heuristic: "Was retrieval result used?" - -``` -After LLM response: -1. Collect all retrieval results from this turn (memory_search, auto-inject L0) -2. For each result, check if any 3+ word substring appears in LLM output -3. Mark used_in_reply = true if found -4. Record metric with used_in_reply flag -``` - -Simple substring check. Not perfect but sufficient for Stage 1 metrics. No LLM call needed. - -##### Implicit Satisfaction Signals - -``` -After user's next message: -1. If message is a correction ("no", "that's wrong", "I meant..."): signal = "correction" -2. If message continues topic (similar embedding): signal = "followup" -3. If message changes topic entirely: signal = "new_topic" -4. If message is appreciation ("thanks", "perfect"): signal = "thanks" -5. Record feedback metric with previous turn's tool context -``` - -Classification via keyword matching + embedding similarity. No LLM call. - -##### Metrics Store Interface - -```go -// EvolutionMetricsStore manages self-evolution metrics. -type EvolutionMetricsStore interface { - // RecordMetric stores a single metric (non-blocking, goroutine-safe). - RecordMetric(ctx context.Context, metric EvolutionMetric) error - - // QueryMetrics returns metrics for analysis. - QueryMetrics(ctx context.Context, agentID uuid.UUID, metricType MetricType, since time.Time, limit int) ([]EvolutionMetric, error) - - // AggregateToolMetrics returns per-tool success rate, avg duration, call count. - AggregateToolMetrics(ctx context.Context, agentID uuid.UUID, since time.Time) ([]ToolAggregate, error) - - // AggregateRetrievalMetrics returns per-source usage rate. - AggregateRetrievalMetrics(ctx context.Context, agentID uuid.UUID, since time.Time) ([]RetrievalAggregate, error) - - // Cleanup removes metrics older than TTL. - Cleanup(ctx context.Context, olderThan time.Time) (int64, error) -} - -type EvolutionMetric struct { - ID uuid.UUID - TenantID uuid.UUID - AgentID uuid.UUID - SessionKey string - MetricType MetricType - MetricKey string - Value json.RawMessage // serialized *MetricValue - CreatedAt time.Time -} - -type ToolAggregate struct { - ToolName string - CallCount int - SuccessRate float64 - AvgDuration time.Duration -} - -type RetrievalAggregate struct { - Source string // "memory_search", "kg_search", "auto_inject" - QueryCount int - UsageRate float64 // fraction of results used in reply - AvgScore float64 -} -``` - -##### Pipeline Integration - -Metrics recorded in ObserveStage (Phase 2 pipeline): - -```go -// In ObserveStage.Execute(): -// After tool result processing: -if toolResult != nil { - go metricsStore.RecordMetric(ctx, EvolutionMetric{ - MetricType: MetricTool, - MetricKey: toolResult.ToolName, - Value: marshalToolMetric(toolResult), - }) -} - -// After retrieval in ContextStage: -if len(l0Results) > 0 { - go metricsStore.RecordMetric(ctx, EvolutionMetric{ - MetricType: MetricRetrieval, - MetricKey: "auto_inject", - Value: marshalRetrievalMetric(query, l0Results), - }) -} -``` - -Always non-blocking (`go` goroutine). Metrics are best-effort — dropped on error, never block agent loop. - -#### Stage 2: Suggestion Engine (v3.1) - -##### DB Schema - -```sql --- New table: agent_evolution_suggestions -CREATE TABLE agent_evolution_suggestions ( - id UUID PRIMARY KEY DEFAULT gen_random_uuid(), - tenant_id UUID NOT NULL REFERENCES tenants(id), - agent_id UUID NOT NULL REFERENCES agents(id) ON DELETE CASCADE, - - suggestion_type TEXT NOT NULL, -- 'threshold', 'tool_order', 'skill_add', 'memory_prune' - suggestion TEXT NOT NULL, -- human-readable suggestion - rationale TEXT NOT NULL, -- data-driven reasoning - parameters JSONB, -- machine-readable params for auto-apply - - status TEXT NOT NULL DEFAULT 'pending', -- 'pending', 'approved', 'rejected', 'applied', 'rolled_back' - reviewed_by TEXT, -- admin user who reviewed - reviewed_at TIMESTAMPTZ, - - created_at TIMESTAMPTZ NOT NULL DEFAULT NOW() -); - -CREATE INDEX idx_evo_suggestions_agent ON agent_evolution_suggestions(agent_id, status); -CREATE INDEX idx_evo_suggestions_tenant ON agent_evolution_suggestions(tenant_id); -``` - -##### Suggestion Types - -```go -type SuggestionType string - -const ( - SuggestThreshold SuggestionType = "threshold" // adjust retrieval threshold - SuggestToolOrder SuggestionType = "tool_order" // reorder tool list for this agent - SuggestSkillAdd SuggestionType = "skill_add" // recommend adding a skill - SuggestMemoryPrune SuggestionType = "memory_prune" // prune low-value memory -) - -// SuggestionParams is the machine-readable payload for auto-apply. -// Each SuggestionType has its own params structure. -type ThresholdParams struct { - Param string `json:"param"` // "retrieval_threshold", "l0_max_tokens" - OldValue float64 `json:"old_value"` - NewValue float64 `json:"new_value"` -} - -type ToolOrderParams struct { - ToolName string `json:"tool_name"` - Action string `json:"action"` // "promote", "demote", "suggest_removal" - Reasoning string `json:"reasoning"` -} -``` - -##### Analysis Rules (Examples) - -``` -Rule 1: Low retrieval usage - IF retrieval.usage_rate < 0.2 over 7 days AND query_count > 50 - THEN suggest: "Raise relevance_threshold from {current} to {current+0.1}" - REASON: "Only 20% of auto-injected memory results appear in responses" - -Rule 2: Tool never succeeds - IF tool.success_rate < 0.1 over 7 days AND call_count > 20 - THEN suggest: "Consider removing {tool_name} or fixing configuration" - REASON: "{tool_name} failed 90% of {call_count} calls" - -Rule 3: Repeated tool usage - IF tool.call_count > 100/week for specific tool AND no skill exists - THEN suggest: "Add skill for {common_task_pattern}" - REASON: "{tool_name} called {count} times this week, mostly for {pattern}" - -Rule 4: Memory bloat - IF episodic_count > 500 AND avg_usage_rate < 0.05 - THEN suggest: "Prune episodic memories older than 60 days" - REASON: "95% of older episodic summaries never retrieved" -``` - -##### Suggestion Store Interface - -```go -type EvolutionSuggestionStore interface { - CreateSuggestion(ctx context.Context, s EvolutionSuggestion) error - ListSuggestions(ctx context.Context, agentID uuid.UUID, status string, limit int) ([]EvolutionSuggestion, error) - UpdateSuggestionStatus(ctx context.Context, id uuid.UUID, status string, reviewedBy string) error - GetSuggestion(ctx context.Context, id uuid.UUID) (*EvolutionSuggestion, error) -} - -type EvolutionSuggestion struct { - ID uuid.UUID - TenantID uuid.UUID - AgentID uuid.UUID - SuggestionType SuggestionType - Suggestion string - Rationale string - Parameters json.RawMessage - Status string - ReviewedBy string - ReviewedAt *time.Time - CreatedAt time.Time -} -``` - -##### Suggestion Engine Interface - -```go -// SuggestionEngine analyzes metrics and generates suggestions. -// Runs as periodic cron job (configurable interval, default: daily). -type SuggestionEngine interface { - // Analyze runs all rules for an agent and creates pending suggestions. - Analyze(ctx context.Context, agentID uuid.UUID) ([]EvolutionSuggestion, error) - - // AnalyzeAll runs analysis for all agents in tenant. - AnalyzeAll(ctx context.Context, tenantID uuid.UUID) error -} - -// RuleSuggestionEngine implements SuggestionEngine with configurable rules. -type RuleSuggestionEngine struct { - metricsStore EvolutionMetricsStore - suggestionStore EvolutionSuggestionStore - rules []AnalysisRule -} - -// AnalysisRule is a single suggestion rule. -type AnalysisRule interface { - // Evaluate checks if the rule fires for the given agent metrics. - // Returns nil if rule doesn't apply. - Evaluate(ctx context.Context, agentID uuid.UUID, metrics AnalysisInput) (*EvolutionSuggestion, error) -} - -type AnalysisInput struct { - ToolAggregates []ToolAggregate - RetrievalAggregates []RetrievalAggregate - FeedbackSummary FeedbackSummary - AnalysisPeriod time.Duration // lookback window (default 7 days) -} -``` - -#### Stage 3: Auto-Adaptation (v3.2+) - -##### Guardrails - -```go -// AdaptationGuardrails control auto-adaptation limits. -type AdaptationGuardrails struct { - // MaxDeltaPerCycle: max parameter change per adaptation cycle. - // E.g., threshold can only change by +/- 0.1 per cycle. - MaxDeltaPerCycle float64 `json:"max_delta_per_cycle"` - - // MinDataPoints: minimum metrics before adaptation allowed. - MinDataPoints int `json:"min_data_points"` - - // RollbackOnDrop: if quality metric drops by this %, rollback. - RollbackOnDrop float64 `json:"rollback_on_drop_pct"` // e.g., 20.0 = 20% - - // AdminOverride: admin can lock parameters (prevent auto-change). - LockedParams []string `json:"locked_params,omitempty"` -} - -// Default guardrails -var DefaultGuardrails = AdaptationGuardrails{ - MaxDeltaPerCycle: 0.1, - MinDataPoints: 100, - RollbackOnDrop: 20.0, -} -``` - -##### Adaptation Flow - -``` -1. SuggestionEngine.Analyze() produces suggestion with status="pending" -2. If auto_adapt enabled for this agent: - a. Check guardrails (min data points, max delta, locked params) - b. If passes: apply parameters, set status="applied" - c. Record baseline quality metric -3. After 7 days: compare quality metric to baseline - a. If quality dropped > RollbackOnDrop%: rollback, set status="rolled_back" - b. If quality stable/improved: keep, emit event -4. Admin can always: approve pending, reject, rollback applied, lock params -``` - -##### Auto-Adapt Store (Extension of Suggestion Store) - -```go -// Added to EvolutionSuggestionStore for Stage 3: -type EvolutionAdaptStore interface { - EvolutionSuggestionStore - - // ApplySuggestion applies a suggestion's parameters to agent config. - ApplySuggestion(ctx context.Context, suggestionID uuid.UUID) error - - // RollbackSuggestion reverts an applied suggestion. - RollbackSuggestion(ctx context.Context, suggestionID uuid.UUID) error - - // GetAdaptationHistory returns applied/rolled-back suggestions for audit. - GetAdaptationHistory(ctx context.Context, agentID uuid.UUID, limit int) ([]EvolutionSuggestion, error) -} -``` - -##### Feature Flags - -```go -// Feature flags in agent settings JSON (agents.settings column) -type SelfEvolutionFlags struct { - MetricsEnabled bool `json:"self_evolution_metrics"` // Stage 1 (default: true for v3) - SuggestionsEnabled bool `json:"self_evolution_suggestions"` // Stage 2 (default: false) - AutoAdaptEnabled bool `json:"self_evolution_auto_adapt"` // Stage 3 (default: false) -} -``` - -No new DB columns. Flags stored in existing `agents.settings` JSONB. - ---- - -### D. EventBus Integration - -#### New Event Topics - -```go -// Orchestration events -const ( - TopicDelegateSent = "delegate:sent" - TopicDelegateCompleted = "delegate:completed" - TopicDelegateFailed = "delegate:failed" -) - -// Self-evolution events -const ( - TopicMetricRecorded = "evolution:metric" - TopicSuggestionCreated = "evolution:suggestion" - TopicAdaptationApplied = "evolution:adapted" - TopicAdaptationRolledBack = "evolution:rollback" -) -``` - -#### Typed Event Payloads - -```go -// DelegateSentPayload emitted when delegate tool fires. -type DelegateSentPayload struct { - SourceAgentID uuid.UUID `json:"source_agent_id"` - TargetAgentID uuid.UUID `json:"target_agent_id"` - LinkID uuid.UUID `json:"link_id"` - Task string `json:"task"` - Mode string `json:"mode"` // "async" or "sync" - SessionKey string `json:"session_key"` -} - -// DelegateCompletedPayload emitted when delegatee finishes. -type DelegateCompletedPayload struct { - SourceAgentID uuid.UUID `json:"source_agent_id"` - TargetAgentID uuid.UUID `json:"target_agent_id"` - LinkID uuid.UUID `json:"link_id"` - Duration time.Duration `json:"duration_ms"` - ToolCalls int `json:"tool_calls"` - Success bool `json:"success"` - Error string `json:"error,omitempty"` -} - -// DelegateFailedPayload emitted on delegation failure. -type DelegateFailedPayload struct { - SourceAgentID uuid.UUID `json:"source_agent_id"` - TargetAgentID uuid.UUID `json:"target_agent_id"` - LinkID uuid.UUID `json:"link_id"` - Reason string `json:"reason"` // "permission_denied", "agent_inactive", "timeout" -} - -// MetricRecordedPayload emitted on each metric write. -type MetricRecordedPayload struct { - AgentID uuid.UUID `json:"agent_id"` - MetricType MetricType `json:"metric_type"` - MetricKey string `json:"metric_key"` -} - -// SuggestionCreatedPayload emitted when suggestion engine generates one. -type SuggestionCreatedPayload struct { - AgentID uuid.UUID `json:"agent_id"` - SuggestionID uuid.UUID `json:"suggestion_id"` - SuggestionType SuggestionType `json:"suggestion_type"` - Summary string `json:"summary"` -} -``` - -#### Event Subscribers - -```go -// Registered at gateway startup: - -// 1. Delegation result delivery — notifies source agent when async delegation completes -bus.Subscribe("delegate-result-delivery", func(e bus.Event) { - if e.Name == TopicDelegateCompleted { - // Deliver result to source agent's session via sessions_send mechanism - } -}) - -// 2. Metrics recording — non-blocking metric persistence -bus.Subscribe("evolution-metrics-recorder", func(e bus.Event) { - if e.Name == TopicMetricRecorded { - // Already persisted by goroutine in ObserveStage. - // Subscriber only for UI real-time updates (WebSocket broadcast). - } -}) - -// 3. Suggestion notification — notify admin of new suggestions -bus.Subscribe("evolution-suggestion-notify", func(e bus.Event) { - if e.Name == TopicSuggestionCreated { - // Broadcast to WebSocket for admin UI notification - } -}) -``` - ---- - -## Related Code Files - -### Files to Modify -| File | Change | -|------|--------| -| `internal/bus/types.go` | Add orchestration + evolution topic constants + payload types | -| `internal/tools/policy.go` | Add applyOrchestrationMode() to FilterTools | -| `internal/agent/loop.go` | Wire metrics recording in observe phase | -| `internal/agent/systemprompt.go` | Add orchestration section to prompt config | -| `internal/store/agent_link_store.go` | (no change — already has full interface) | -| `cmd/gateway.go` | Register delegate tool, event subscribers, metrics cron | - -### Files to Create -| File | Purpose | -|------|---------| -| `internal/tools/delegate_tool.go` | DelegateTool implementation | -| `internal/tools/delegate_workspace.go` | DelegateWorkspaceConfig, workspace setup | -| `internal/agent/orchestration_mode.go` | OrchestrationMode types, resolution logic | -| `internal/agent/agent_runner.go` | AgentRunner interface for delegation dispatch | -| `internal/store/evolution_store.go` | EvolutionMetricsStore + EvolutionSuggestionStore interfaces | -| `internal/store/pg/evolution_metrics.go` | PG implementation of EvolutionMetricsStore | -| `internal/store/pg/evolution_suggestions.go` | PG implementation of EvolutionSuggestionStore | -| `internal/agent/evolution_metrics.go` | Metric types + recording helpers | -| `internal/agent/evolution_suggestion_engine.go` | SuggestionEngine + AnalysisRule implementations | -| `internal/agent/evolution_guardrails.go` | AdaptationGuardrails + auto-adapt logic | - ---- - -## Design Steps - -1. Define DelegateTool interface + parameters schema -2. Design delegate execution flow (async + sync modes) -3. Define DelegateWorkspaceConfig in agent_link.settings -4. Define AgentRunner interface for delegation dispatch -5. Define OrchestrationMode enum + resolution logic (tenant -> agent) -6. Design tool visibility filtering by orchestration mode -7. Design orchestration section for system prompt template -8. Define agent_evolution_metrics schema + indexes -9. Define metric types (retrieval, tool, feedback) with value structs -10. Define EvolutionMetricsStore interface + aggregate methods -11. Design metrics recording integration in ObserveStage -12. Define agent_evolution_suggestions schema -13. Define SuggestionEngine + AnalysisRule interfaces -14. Design analysis rules (4 initial rules) -15. Define AdaptationGuardrails + auto-adapt flow -16. Define all EventBus topics + typed payloads -17. Design event subscriber wiring at startup - ---- - -## Todo List - -- [ ] A1: DelegateTool interface + parameters -- [ ] A2: Delegate execution flow (async/sync) -- [ ] A3: DelegateWorkspaceConfig schema -- [ ] A4: AgentRunner interface -- [ ] A5: Permission flow (CanDelegate + GetLinkBetween) -- [ ] B1: OrchestrationMode enum + constants -- [ ] B2: Mode resolution (tenant default + agent override) -- [ ] B3: Tool visibility by mode in PolicyEngine -- [ ] B4: Orchestration section in prompt template -- [ ] C1: agent_evolution_metrics DDL + indexes -- [ ] C2: Metric value types (retrieval, tool, feedback) -- [ ] C3: EvolutionMetricsStore interface -- [ ] C4: Retrieval "used in reply" heuristic -- [ ] C5: Implicit satisfaction signal classification -- [ ] C6: agent_evolution_suggestions DDL -- [ ] C7: SuggestionEngine + AnalysisRule interfaces -- [ ] C8: Initial 4 analysis rules -- [ ] C9: AdaptationGuardrails + auto-adapt flow -- [ ] C10: Feature flags in agent settings -- [ ] D1: EventBus topic constants -- [ ] D2: Typed event payloads (delegate, evolution) -- [ ] D3: Event subscriber design (result delivery, notifications) - ---- - -## Success Criteria - -1. DelegateTool ~100 LOC (excluding workspace setup), uses existing agentLinkStore.CanDelegate() -2. Orchestration mode resolvable from tenant + agent settings with no new DB columns -3. PolicyEngine correctly filters tools by mode (spawn/delegate/team) -4. agent_evolution_metrics table handles 10K+ rows/day without index degradation -5. Metrics recording never blocks agent loop (goroutine + best-effort) -6. SuggestionEngine rules are additive (new rule = implement AnalysisRule interface) -7. All orchestration events typed (no `map[string]any` payloads) -8. Feature flags independently toggleable per agent - ---- - -## Risk Assessment - -| Risk | Severity | Mitigation | -|------|----------|------------| -| Delegate tool workspace isolation leaks | HIGH | ReadOnlyPaths enforced by same resolvePath() as all file tools. SharedPath created with restricted permissions. | -| Sync delegation timeout blocks caller | MEDIUM | Configurable timeout per link (default 300s). Timeout returns error, does not kill delegatee. | -| Metrics table grows unbounded | MEDIUM | TTL cleanup cron (default 90 days). Partitioning-ready schema (created_at). Monitor table size. | -| Suggestion engine generates noise | LOW | Minimum data points threshold (100). Suggestions require admin review (Stage 2). Auto-adapt has guardrails (Stage 3). | -| Orchestration mode config not discoverable | LOW | Admin UI shows current mode per agent. CLI: goclaw agent info shows mode. | -| Self-evolution "learns wrong" | HIGH | Progressive rollout. Stage 1 = metrics only (no behavior change). Stage 2 = human review. Stage 3 = guardrails + rollback. | -| EventBus topic explosion | LOW | Strict naming convention. Only add topics with subscribers. No orphan topics. | - ---- - -## Security Considerations - -- **Delegation permission:** `CanDelegate()` is the sole gate. No bypass via direct tool call — delegate tool always calls CanDelegate(). Link status "disabled" blocks delegation. -- **Cross-tenant delegation:** Agent links are tenant-scoped (both agents must be in same tenant). `agentLinkStore` queries filter by tenant_id from context. -- **Delegate workspace isolation:** Delegatee cannot access delegator's full workspace. Only explicitly exported paths (SourceExports) are shared read-only. SharedPath is a new directory, not inside either workspace. -- **Metrics data sensitivity:** Metrics contain query text and tool names, not user data or PII. Scoped by tenant_id. Cleanup cron prevents accumulation. -- **Auto-adapt scope:** Stage 3 only modifies retrieval parameters (thresholds, weights). Cannot modify tool access, workspace rules, or security settings. Admin can lock any parameter. diff --git a/plans/260406-1304-goclaw-v3-core-design/phase-06-migration-audit.md b/plans/260406-1304-goclaw-v3-core-design/phase-06-migration-audit.md deleted file mode 100644 index 582ffe40..00000000 --- a/plans/260406-1304-goclaw-v3-core-design/phase-06-migration-audit.md +++ /dev/null @@ -1,683 +0,0 @@ -# Phase 6: Migration & Audit - -## Context Links -- Brainstorm: `plans/reports/brainstorm-260406-1059-goclaw-v3-core-redesign.md` (sections 16, 17) -- Architecture report: `plans/reports/Explore-260406-thorough-v3-architecture.md` (all sections) -- Workspace forcing: `plans/reports/Explore-260406-workspace-forcing.md` -- Phase 1-5 designs: all prior phase files in this plan -- Current migrations: `migrations/000001_init_schema.up.sql` through `migrations/000036_secure_cli_agent_grants.up.sql` -- Schema version: `internal/upgrade/version.go` (RequiredSchemaVersion = 36) - -## Overview -- **Priority:** P1 (blocks implementation start) -- **Status:** Complete -- **Effort:** 4h design - -Final design phase — comprehensive migration plan, affected file inventory, test matrix, and audit checklist. Ensures no implementation begins without full impact analysis. - ---- - -## Key Insights - -1. Current RequiredSchemaVersion = 36 (migration 000036). V3 adds 3 new tables + 2 ALTER TABLE. Next migration = 000037. -2. KG tables (000013) lack temporal columns. Adding valid_from/valid_until via ALTER is non-breaking — NULL defaults work. -3. Memory system is hybrid: daily .md files in filesystem + memory_documents/memory_chunks in PG. V3 must ingest filesystem files into new episodic_summaries table. -4. Agent settings already uses JSONB — all new per-agent configs (retrieval thresholds, orchestration mode, self-evolution flags, dedup thresholds) fit without schema changes. -5. No SQLite changes in v3 — design is PG-first per brainstorm decision #17. SQLite compat deferred. - ---- - -## Requirements - -### Functional -- F1: New tables created via standard golang-migrate files -- F2: ALTER TABLE for KG temporal columns with safe defaults -- F3: Lazy data migration: v2 data auto-converted on first access -- F4: Feature flags per agent for gradual v3 rollout -- F5: Full affected file inventory with dependency order -- F6: Integration test plan covering all 6 workspace scenarios -- F7: Rollback path for every schema change - -### Non-Functional -- NF1: Migration runs <30s on databases with 100K entities -- NF2: No downtime required — all changes additive (no DROP/RENAME) -- NF3: v2 agents continue working unchanged until explicitly migrated -- NF4: Test suite runnable in CI (<5 min) - ---- - -## Architecture - -### A. DB Schema Migration - -#### Migration 000037: v3 memory + evolution tables - -```sql --- 000037_v3_memory_evolution.up.sql - --- Episodic summaries (Tier 2 memory) -CREATE TABLE episodic_summaries ( - id UUID PRIMARY KEY DEFAULT gen_random_uuid(), - tenant_id UUID NOT NULL REFERENCES tenants(id), - agent_id UUID NOT NULL REFERENCES agents(id) ON DELETE CASCADE, - user_id VARCHAR(255) NOT NULL DEFAULT '', - session_key TEXT NOT NULL, - - summary TEXT NOT NULL, -- LLM-generated session summary - key_topics TEXT[] DEFAULT '{}', -- extracted topic keywords - embedding vector(1536), -- for similarity search - source_type TEXT NOT NULL DEFAULT 'session', -- 'session', 'v2_daily', 'manual' - source_id TEXT, -- session key or v2 file path (dedup guard) - turn_count INT NOT NULL DEFAULT 0, -- number of turns summarized - token_count INT NOT NULL DEFAULT 0, -- estimated tokens in original - - created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), - expires_at TIMESTAMPTZ -- NULL = no expiry. TTL configurable per agent. -); - -CREATE INDEX idx_episodic_agent_user ON episodic_summaries(agent_id, user_id); -CREATE INDEX idx_episodic_tenant ON episodic_summaries(tenant_id); -CREATE INDEX idx_episodic_source ON episodic_summaries(agent_id, source_id); -CREATE INDEX idx_episodic_tsv ON episodic_summaries USING GIN(to_tsvector('simple', summary)); -CREATE INDEX idx_episodic_vec ON episodic_summaries USING hnsw(embedding vector_cosine_ops); -CREATE INDEX idx_episodic_expires ON episodic_summaries(expires_at) WHERE expires_at IS NOT NULL; - --- Evolution metrics (Stage 1 self-evolution) -CREATE TABLE agent_evolution_metrics ( - id UUID PRIMARY KEY DEFAULT gen_random_uuid(), - tenant_id UUID NOT NULL REFERENCES tenants(id), - agent_id UUID NOT NULL REFERENCES agents(id) ON DELETE CASCADE, - session_key TEXT NOT NULL, - - metric_type TEXT NOT NULL, -- 'retrieval', 'tool', 'feedback' - metric_key TEXT NOT NULL, -- tool name, retrieval source, signal type - value JSONB NOT NULL, -- metric-specific payload - - created_at TIMESTAMPTZ NOT NULL DEFAULT NOW() -); - -CREATE INDEX idx_evo_metrics_agent_type ON agent_evolution_metrics(agent_id, metric_type); -CREATE INDEX idx_evo_metrics_created ON agent_evolution_metrics(created_at); -CREATE INDEX idx_evo_metrics_tenant ON agent_evolution_metrics(tenant_id); - --- Evolution suggestions (Stage 2 self-evolution) -CREATE TABLE agent_evolution_suggestions ( - id UUID PRIMARY KEY DEFAULT gen_random_uuid(), - tenant_id UUID NOT NULL REFERENCES tenants(id), - agent_id UUID NOT NULL REFERENCES agents(id) ON DELETE CASCADE, - - suggestion_type TEXT NOT NULL, -- 'threshold', 'tool_order', 'skill_add', 'memory_prune' - suggestion TEXT NOT NULL, - rationale TEXT NOT NULL, - parameters JSONB, -- machine-readable for auto-apply - - status TEXT NOT NULL DEFAULT 'pending', -- 'pending','approved','rejected','applied','rolled_back' - reviewed_by TEXT, - reviewed_at TIMESTAMPTZ, - - created_at TIMESTAMPTZ NOT NULL DEFAULT NOW() -); - -CREATE INDEX idx_evo_suggestions_agent ON agent_evolution_suggestions(agent_id, status); -CREATE INDEX idx_evo_suggestions_tenant ON agent_evolution_suggestions(tenant_id); -``` - -**Down migration:** -```sql --- 000037_v3_memory_evolution.down.sql -DROP TABLE IF EXISTS agent_evolution_suggestions; -DROP TABLE IF EXISTS agent_evolution_metrics; -DROP TABLE IF EXISTS episodic_summaries; -``` - -#### Migration 000038: KG temporal columns - -```sql --- 000038_kg_temporal_columns.up.sql - --- Add temporal validity windows to KG entities and relations -ALTER TABLE kg_entities ADD COLUMN IF NOT EXISTS valid_from TIMESTAMPTZ DEFAULT NOW(); -ALTER TABLE kg_entities ADD COLUMN IF NOT EXISTS valid_until TIMESTAMPTZ; -- NULL = current/valid - -ALTER TABLE kg_relations ADD COLUMN IF NOT EXISTS valid_from TIMESTAMPTZ DEFAULT NOW(); -ALTER TABLE kg_relations ADD COLUMN IF NOT EXISTS valid_until TIMESTAMPTZ; - --- Index for temporal queries: "current facts" (most common query) -CREATE INDEX idx_kg_entities_current ON kg_entities(agent_id, user_id) - WHERE valid_until IS NULL; - --- Index for historical queries: "facts valid at time T" -CREATE INDEX idx_kg_entities_temporal ON kg_entities(agent_id, user_id, valid_from, valid_until); - -CREATE INDEX idx_kg_relations_current ON kg_relations(agent_id, user_id) - WHERE valid_until IS NULL; - -CREATE INDEX idx_kg_relations_temporal ON kg_relations(agent_id, user_id, valid_from, valid_until); -``` - -**Down migration:** -```sql --- 000038_kg_temporal_columns.down.sql -DROP INDEX IF EXISTS idx_kg_relations_temporal; -DROP INDEX IF EXISTS idx_kg_relations_current; -DROP INDEX IF EXISTS idx_kg_entities_temporal; -DROP INDEX IF EXISTS idx_kg_entities_current; - -ALTER TABLE kg_relations DROP COLUMN IF EXISTS valid_until; -ALTER TABLE kg_relations DROP COLUMN IF EXISTS valid_from; -ALTER TABLE kg_entities DROP COLUMN IF EXISTS valid_until; -ALTER TABLE kg_entities DROP COLUMN IF EXISTS valid_from; -``` - -#### RequiredSchemaVersion Bump - -```go -// internal/upgrade/version.go -const RequiredSchemaVersion uint = 38 -``` - ---- - -### B. Data Migration Strategy (Dual-Mode) - -#### Principle: Lazy Migration on First Access - -V3 reads v2 format on first access, auto-migrates in background. No bulk migration script. No downtime. - -#### V2 Memory Files -> Episodic Summaries - -``` -Trigger: Agent with v3_memory_enabled=true, first session start -Flow: -1. Check if agent has unmigrated v2 daily files: - - List files matching memory/YYYY-MM-DD.md in workspace - - Check episodic_summaries for source_id matching file path - - If no matching episodic record: file is unmigrated -2. For each unmigrated file: - a. Read file content - b. Insert episodic_summaries row: - - summary: file content (already summarized by v2 flush) - - source_type: "v2_daily" - - source_id: file path (dedup guard) - - key_topics: extract from content (simple keyword extraction) - - turn_count: 0 (unknown for v2) - - token_count: len(content) / 4 (estimate) - c. Generate embedding in background (non-blocking) -3. Mark migration complete in agent settings: v2_memory_migrated=true -``` - -#### V2 KG Entities -> Temporal - -``` -Trigger: Migration 000038 runs (ALTER TABLE with defaults) -Flow: -1. ALTER TABLE adds valid_from DEFAULT NOW() — new rows get current timestamp -2. Existing rows: valid_from = NOW() at migration time (not created_at) -3. Optional backfill (non-blocking, cron job): - UPDATE kg_entities SET valid_from = created_at WHERE valid_from > created_at + INTERVAL '1 second'; - UPDATE kg_relations SET valid_from = created_at WHERE valid_from > created_at + INTERVAL '1 second'; -4. valid_until remains NULL for all existing entities (all considered "current") -``` - -> **Decision:** Default valid_from to NOW() at migration time, not created_at. Backfill is optional — historical accuracy is nice-to-have, not required. Existing entities are "currently valid" regardless of when they were created. - -#### V2 Sessions -> V3 Sessions - -No migration needed. V3 pipeline operates on same session data. Sessions use same `sessions` table, same `SessionData` struct. V3 adds pipeline stages that wrap existing behavior. - -#### Feature Flags - -All stored in existing `agents.settings` JSONB column: - -```json -{ - "v3_pipeline_enabled": false, - "v3_memory_enabled": false, - "v3_retrieval_enabled": false, - "self_evolution_metrics": false, - "self_evolution_suggestions": false, - "self_evolution_auto_adapt": false, - "v2_memory_migrated": false, - "retrieval_threshold": 0.3, - "dedup_auto_merge_threshold": 0.98, - "dedup_flag_threshold": 0.90, - "orchestration_mode": "spawn" -} -``` - -**Rollout plan:** -1. Deploy v3 binary with all flags false (default) — no behavior change -2. Enable v3_memory_enabled per agent — triggers lazy v2 file migration -3. Enable v3_pipeline_enabled per agent — switches to Stage-based pipeline -4. Enable v3_retrieval_enabled per agent — activates L0 auto-inject -5. Enable self_evolution_metrics per agent — starts recording -6. Enable at tenant level for bulk rollout - ---- - -### C. Code Migration Plan — Affected Files Inventory - -#### Module 1: Foundation (Phase 1) -| File | Change Type | Dependency | -|------|-------------|------------| -| `internal/agent/loop_types.go` | MODIFY: split RunState into substates | None | -| `internal/agent/loop_compact.go` | MODIFY: replace chars/4 with TokenCounter | TokenCounter | -| `internal/agent/pruning.go` | MODIFY: use TokenCounter for thresholds | TokenCounter | -| `internal/tools/context_keys.go` | MODIFY: replace dual workspace keys with WorkspaceContext | None | -| `internal/tools/workspace_resolver.go` | MODIFY: return WorkspaceContext instead of string | None | -| `internal/tools/workspace_dir.go` | MODIFY: use WorkspaceContext | WorkspaceContext | -| `internal/bus/types.go` | MODIFY: add new event topics + typed payloads | None | -| `internal/bus/bus.go` | No change (existing Broadcast mechanism sufficient) | None | -| `internal/providers/types.go` | MODIFY: add ProviderCapabilities struct | None | -| `internal/providers/anthropic.go` | MODIFY: implement ProviderAdapter interface | ProviderAdapter | -| `internal/providers/openai.go` | MODIFY: implement ProviderAdapter interface | ProviderAdapter | -| `internal/providers/dashscope.go` | MODIFY: implement ProviderAdapter interface | ProviderAdapter | -| `internal/providers/codex.go` | MODIFY: implement ProviderAdapter interface | ProviderAdapter | -| `internal/providers/claude_cli.go` | MODIFY: implement ProviderAdapter interface | ProviderAdapter | -| `internal/providers/acp_provider.go` | MODIFY: implement ProviderAdapter interface | ProviderAdapter | - -**New files:** -| File | Purpose | -|------|---------| -| `internal/agent/token_counter.go` | TokenCounter interface + tiktoken impl | -| `internal/tools/workspace_context.go` | WorkspaceContext struct + helpers | -| `internal/providers/adapter.go` | ProviderAdapter interface | -| `internal/providers/capabilities.go` | ProviderCapabilities struct | - -#### Module 2: Pipeline (Phase 2) -| File | Change Type | Dependency | -|------|-------------|------------| -| `internal/agent/loop.go` | MAJOR: refactor into pipeline stages | Phase 1 foundation | -| `internal/agent/loop_run.go` | MODIFY: pipeline entry point | Pipeline stages | -| `internal/agent/loop_context.go` | MODIFY: becomes ContextStage | WorkspaceContext | -| `internal/agent/loop_history.go` | MODIFY: becomes part of ContextStage | PromptBuilder | -| `internal/agent/loop_tools.go` | MODIFY: becomes ToolStage | ToolRegistry | -| `internal/agent/toolloop.go` | MODIFY: integrated into ToolStage | ToolStage | -| `internal/agent/loop_compact.go` | MODIFY: becomes PruneStage | TokenCounter | -| `internal/agent/loop_finalize.go` | MODIFY: becomes FinalizeStage | EventBus | -| `internal/agent/loop_tracing.go` | MODIFY: per-stage tracing | Pipeline stages | - -**New files:** -| File | Purpose | -|------|---------| -| `internal/agent/pipeline.go` | Pipeline, Stage interface, runner | -| `internal/agent/stage_context.go` | ContextStage | -| `internal/agent/stage_think.go` | ThinkStage (LLM call) | -| `internal/agent/stage_prune.go` | PruneStage (token counting + pruning) | -| `internal/agent/stage_tool.go` | ToolStage (tool execution) | -| `internal/agent/stage_observe.go` | ObserveStage (loop detection, metrics) | -| `internal/agent/stage_checkpoint.go` | CheckpointStage (session save) | - -#### Module 3: Memory & KG (Phase 3) -| File | Change Type | Dependency | -|------|-------------|------------| -| `internal/memory/embeddings.go` | MODIFY: integrate with episodic store | EpisodicStore | -| `internal/knowledgegraph/extractor.go` | MODIFY: async extraction via events | EventBus | -| `internal/knowledgegraph/extractor_prompt.go` | No change | | -| `internal/store/memory_store.go` | MODIFY: add EpisodicSummaryStore interface | None | -| `internal/store/knowledge_graph_store.go` | MODIFY: add temporal query methods | None | -| `internal/store/pg/memory_docs.go` | No change (v2 compat) | | -| `internal/store/pg/memory_search.go` | MODIFY: integrate episodic search | EpisodicStore | -| `internal/store/pg/knowledge_graph.go` | MODIFY: temporal WHERE clauses | KG temporal cols | -| `internal/store/pg/knowledge_graph_dedup.go` | MODIFY: configurable thresholds | Agent settings | -| `internal/store/pg/knowledge_graph_relations.go` | MODIFY: temporal WHERE clauses | KG temporal cols | -| `internal/store/pg/knowledge_graph_traversal.go` | MODIFY: temporal filter in traversal | KG temporal cols | -| `internal/store/pg/knowledge_graph_embedding.go` | No change | | -| `internal/agent/memoryflush.go` | MODIFY: v3 path uses event-driven consolidation | EventBus | -| `internal/agent/extractive_memory.go` | No change (fallback for v2 mode) | | - -**New files:** -| File | Purpose | -|------|---------| -| `internal/store/episodic_store.go` | EpisodicSummaryStore interface | -| `internal/store/pg/episodic_summaries.go` | PG implementation | -| `internal/agent/consolidation_worker.go` | Event-driven session -> episodic -> semantic | -| `internal/agent/memory_migrator.go` | Lazy v2 daily files -> episodic migration | - -#### Module 4: System Integration (Phase 4) -| File | Change Type | Dependency | -|------|-------------|------------| -| `internal/agent/systemprompt.go` | MAJOR: replace with PromptBuilder | PromptConfig types | -| `internal/agent/systemprompt_sections.go` | REMOVE: logic moves to template | PromptBuilder | -| `internal/tools/types.go` | MODIFY: add Metadata() to Tool interface | None | -| `internal/tools/registry.go` | MODIFY: store ToolMetadata | ToolMetadata | -| `internal/tools/policy.go` | MODIFY: capability rules + role param | ToolCapability | -| `internal/tools/filesystem.go` | MODIFY: WorkspaceContextFromCtx() | WorkspaceContext | -| `internal/tools/filesystem_write.go` | MODIFY: WorkspaceContextFromCtx() | WorkspaceContext | -| `internal/tools/filesystem_list.go` | MODIFY: WorkspaceContextFromCtx() | WorkspaceContext | -| `internal/tools/shell.go` | MODIFY: WorkspaceContextFromCtx() | WorkspaceContext | -| `internal/tools/edit.go` | MODIFY: WorkspaceContextFromCtx() | WorkspaceContext | -| `internal/tools/memory.go` | MODIFY: add depth parameter | L0/L1/L2 | -| `internal/tools/knowledge_graph.go` | MODIFY: add as_of temporal param | KG temporal | -| `internal/tools/skill_search.go` | MODIFY: hybrid search integration | HybridSearcher | -| `internal/skills/loader.go` | MODIFY: ACL-aware BuildSummary() | SkillAccessStore | -| `internal/skills/search.go` | MODIFY: expose for fusion with embedding | None | - -**New files:** (listed in Phase 4 design) - -#### Module 5: Orchestration & Intelligence (Phase 5) -| File | Change Type | Dependency | -|------|-------------|------------| -| `internal/tools/policy.go` | MODIFY: orchestration mode filtering | OrchestrationMode | -| `internal/bus/types.go` | MODIFY: delegation + evolution event types | None | -| `internal/agent/loop.go` | MODIFY: metrics recording hooks | EvolutionMetrics | -| `cmd/gateway.go` | MODIFY: register delegate tool, event subscribers | All Phase 5 | - -**New files:** (listed in Phase 5 design) - -#### Breaking Changes Inventory - -| Change | Impact | Backward Compat | -|--------|--------|-----------------| -| Tool.Metadata() added to interface | All tool implementations | Wrapper: `defaultMetadata(tool)` | -| WorkspaceContext replaces dual keys | All file tools + loop_context | Single migration: grep `ToolWorkspaceFromCtx` (15 sites) | -| PromptBuilder replaces BuildSystemPrompt() | loop_history.go, tests | Bridge: old func calls PromptBuilder.Build() | -| Pipeline stages replace runLoop() | loop.go (804 lines) | Feature flag: v3_pipeline_enabled per agent | -| TokenCounter replaces chars/4 | loop_compact.go, pruning.go | Fallback: chars/4 for unknown models | -| KG temporal columns | All KG queries | Default: WHERE valid_until IS NULL (same as pre-temporal) | -| Episodic summaries table | New (no existing code depends on it) | Fully additive | -| Evolution metrics/suggestions | New (no existing code depends on it) | Fully additive | - ---- - -### D. Integration Test Plan - -#### Test Matrix: Workspace Scenarios - -From workspace forcing report — 6 scenarios must pass: - -| # | Scenario | ActivePath | Scope | Key Assertion | -|---|----------|------------|-------|---------------| -| 1 | Solo DM, isolated | `{base}/{userID}` | personal | Files scoped to user dir | -| 2 | Solo DM, shared | `{base}` | personal | Files in shared agent dir | -| 3 | Team dispatch (member) | `{base}/teams/{teamID}/{chatID}` | team-shared | Workspace forced to team dir | -| 4 | Team dispatch (leader) | `{base}/teams/{teamID}/{chatID}` | team-shared | Leader sees team workspace | -| 5 | Team inbound (member) | `{base}/teams/{teamID}/{chatID}` | team-isolated | Per-chat isolation within team | -| 6 | Delegate workspace | delegatee workspace + shared area | delegate | Read-only exports enforced | - -#### Memory Tier Tests - -| Test | Input | Expected | -|------|-------|----------| -| Working memory (Tier 1) | 20 messages in session | All messages in context window | -| Episodic creation | Session end event | episodic_summaries row created | -| Episodic dedup | Same session end event replayed | No duplicate (source_id unique) | -| Semantic extraction | Episodic with entities | KG entities upserted | -| L0 auto-inject | User message with relevant topic | Memory section in system prompt | -| L0 skip | "hello" message (low relevance) | No Memory section | -| V2 migration | Agent with memory/2026-01-01.md | Episodic row with source_type=v2_daily | -| TTL expiry | Episodic with expires_at in past | Cleaned by cron | - -#### KG Temporal Tests - -| Test | Input | Expected | -|------|-------|----------| -| New entity | UpsertEntity() | valid_from=NOW(), valid_until=NULL | -| Supersede entity | Contradicting fact | Old: valid_until=NOW(). New: valid_from=NOW() | -| Current query | SearchEntities() default | Only valid_until IS NULL entities | -| Historical query | SearchEntities(as_of="2026-01-01") | Entities valid at that date | -| Temporal traversal | Traverse() with temporal entities | Only current relations followed | -| Configurable dedup | Agent with threshold 0.95 | Auto-merge at 0.95, not hardcoded 0.98 | - -#### Pipeline Stage Tests - -| Test | Stage | Input | Expected | -|------|-------|-------|----------| -| Context build | ContextStage | RunRequest + agent config | WorkspaceContext resolved, system prompt built | -| LLM call | ThinkStage | Messages array | Provider called via adapter | -| Token pruning | PruneStage | Messages exceeding 85% context | Older messages pruned, token count accurate | -| Tool exec | ToolStage | Tool call from LLM | Tool executed via registry with policy check | -| Loop detect | ObserveStage | 3x same tool call | Loop detected, force stop | -| Session save | CheckpointStage | Completed run | Session persisted, events emitted | -| Full pipeline | All stages | Complete request | End-to-end: request -> response -> session saved | - -#### Provider Adapter Tests - -| Test | Provider | Input | Expected | -|------|----------|-------|----------| -| Anthropic wire | Anthropic | Internal ChatRequest | Valid Anthropic API JSON | -| OpenAI wire | OpenAI | Internal ChatRequest | Valid OpenAI API JSON | -| DashScope no-stream-tools | DashScope | ChatRequest with tools | Streaming disabled, tools present | -| Codex OAuth | Codex | ChatRequest | OAuth token refreshed if needed | -| Claude CLI | ClaudeCLI | ChatRequest | Stdio command built correctly | -| Capabilities | All providers | Capabilities() | Correct flags per provider | -| Round-trip | All providers | ToRequest -> FromResponse | Internal format preserved | - -#### Regression Tests - -| Test | Assertion | -|------|-----------| -| V2 agent with all flags false | Identical behavior to pre-v3 binary | -| V2 memory flush | Daily .md files still written when v3_memory_enabled=false | -| V2 KG without temporal | Queries return same results (WHERE valid_until IS NULL matches all) | -| V2 system prompt | PromptBuilder with v2 config produces identical output | -| V2 tool registry | Tools without Metadata() get default metadata wrapper | - ---- - -### E. Audit Checklist - -#### Interface Contracts -- [ ] All interface contracts have method signatures with ctx context.Context as first param -- [ ] All interfaces have at least one implementation (PG store) -- [ ] No interface has >10 methods (split if so) -- [ ] Error returns are typed (not raw string errors) -- [ ] Context propagation: tenant_id, agent_id, user_id carried through all store calls - -#### DB Schemas -- [ ] All new tables have tenant_id column with FK to tenants(id) -- [ ] All FKs have ON DELETE CASCADE where appropriate -- [ ] Indexes exist for: primary lookup, tenant filter, time-range queries -- [ ] No new columns on existing tables (all config in JSONB settings) -- [ ] Down migrations cleanly reverse up migrations -- [ ] No PG-specific features in interface definitions (SQLite compat future) - -#### Data Flow -- [ ] Memory flow: user message -> auto-inject L0 -> response -> session end -> episodic -> semantic -- [ ] KG flow: text -> extract -> upsert -> dedup -> temporal validity -- [ ] Pipeline flow: ContextStage -> ThinkStage -> PruneStage -> ToolStage -> ObserveStage -> CheckpointStage -- [ ] Delegation flow: delegate() -> permission check -> workspace setup -> AgentRunner -> result delivery -- [ ] Event flow: stage completion -> EventBus.Broadcast() -> subscriber handlers - -#### Breaking Changes -- [ ] Tool.Metadata() — wrapper provided for existing tools -- [ ] WorkspaceContext — single context key replaces 2 (grep all call sites) -- [ ] BuildSystemPrompt() — bridge function wraps PromptBuilder -- [ ] Pipeline stages — feature flag gates new behavior -- [ ] Token counting — fallback to chars/4 for unknown models - -#### Security Review -- [ ] Workspace isolation: WorkspaceContext.ActivePath enforced by resolvePath() -- [ ] Delegate read-only exports: validated against delegator's workspace boundary -- [ ] Delegate shared area: created with restrictive permissions (0750) -- [ ] Tenant boundary: all new tables have tenant_id, all queries filter by it -- [ ] Metrics data: no PII stored (queries + tool names, not user content) -- [ ] Template injection: PromptConfig data passed via {{ printf "%s" }} not {{ }} -- [ ] Auto-adapt scope: only modifies retrieval params, not security settings - -#### Performance Review -- [ ] tiktoken-go: benchmark Count() for Claude models (<1ms per 1000 tokens) -- [ ] pgvector: benchmark episodic_summaries search at 100K rows (<50ms) -- [ ] Event bus: benchmark Broadcast throughput (10K events/sec target) -- [ ] Template rendering: benchmark Build() (<1ms, no allocation-heavy paths) -- [ ] L0 auto-inject: benchmark memory index query (<10ms per turn) -- [ ] KG temporal: benchmark index scan with valid_until IS NULL filter -- [ ] Metrics recording: verify non-blocking (goroutine, no mutex on hot path) - ---- - -## Related Code Files - -### Migration Files to Create -| File | Purpose | -|------|---------| -| `migrations/000037_v3_memory_evolution.up.sql` | episodic_summaries, agent_evolution_metrics, agent_evolution_suggestions | -| `migrations/000037_v3_memory_evolution.down.sql` | Drop above tables | -| `migrations/000038_kg_temporal_columns.up.sql` | ALTER kg_entities/kg_relations add valid_from/valid_until | -| `migrations/000038_kg_temporal_columns.down.sql` | DROP added columns + indexes | - -### Files to Modify -| File | Change | -|------|--------| -| `internal/upgrade/version.go` | RequiredSchemaVersion = 38 | - -### Full Affected File Count by Module - -| Module | Files Modified | Files Created | Total | -|--------|---------------|---------------|-------| -| Foundation (Phase 1) | 15 | 4 | 19 | -| Pipeline (Phase 2) | 9 | 7 | 16 | -| Memory & KG (Phase 3) | 14 | 4 | 18 | -| System Integration (Phase 4) | 15 | 9 | 24 | -| Orchestration (Phase 5) | 4 | 10 | 14 | -| Migration (Phase 6) | 1 | 4 | 5 | -| **Total** | **58** | **38** | **96** | - ---- - -## Design Steps - -1. Write migration 000037 DDL (episodic_summaries, evolution tables) -2. Write migration 000038 DDL (KG temporal columns) -3. Write down migrations for both -4. Document lazy v2 memory migration flow -5. Document KG temporal backfill strategy -6. List all feature flags with defaults -7. Inventory all affected files per module (complete) -8. Document breaking changes with migration path -9. Write workspace scenario test matrix -10. Write memory tier test scenarios -11. Write KG temporal test scenarios -12. Write pipeline stage test scenarios -13. Write provider adapter test matrix -14. Write regression test list -15. Complete audit checklist - ---- - -## Todo List - -- [ ] A1: Migration 000037 DDL (3 new tables) -- [ ] A2: Migration 000038 DDL (KG temporal ALTER) -- [ ] A3: Down migrations for both -- [ ] A4: RequiredSchemaVersion bump to 38 -- [ ] B1: V2 memory -> episodic lazy migration design -- [ ] B2: KG temporal backfill strategy -- [ ] B3: Feature flags catalog with defaults -- [ ] B4: Session v2->v3 compatibility notes -- [ ] C1: Affected files inventory per module (complete) -- [ ] C2: Code change dependency order -- [ ] C3: Breaking changes list with backward-compat path -- [ ] D1: Workspace scenario test matrix (6 scenarios) -- [ ] D2: Memory tier tests (8 scenarios) -- [ ] D3: KG temporal tests (6 scenarios) -- [ ] D4: Pipeline stage tests (7 scenarios) -- [ ] D5: Provider adapter tests (7 scenarios) -- [ ] D6: Regression tests (5 scenarios) -- [ ] E1: Interface contracts audit -- [ ] E2: DB schema audit -- [ ] E3: Data flow audit -- [ ] E4: Breaking changes audit -- [ ] E5: Security review -- [ ] E6: Performance review - ---- - -## Success Criteria - -1. Both migrations run cleanly on empty DB and on DB with existing v2 data -2. Down migrations fully reverse up migrations (no orphaned objects) -3. RequiredSchemaVersion matches highest migration number -4. All 6 workspace scenarios pass integration tests -5. V2 agents with all flags false behave identically to pre-v3 -6. Every breaking change has documented backward-compat path -7. Audit checklist 100% complete before implementation begins - ---- - -## Risk Assessment - -| Risk | Severity | Mitigation | -|------|----------|------------| -| Migration fails on large KG tables | MEDIUM | ALTER TABLE ADD COLUMN with DEFAULT is metadata-only in PG12+ (instant). No table rewrite. | -| V2 memory files not found (path differences) | LOW | Lazy migration lists actual files in workspace. Files missing = nothing to migrate. | -| Feature flag combinatorial explosion | MEDIUM | 6 boolean flags = 64 combos. Test key combos only: all-off (v2), all-on (v3), individual toggles. | -| 96 affected files scope | HIGH | Implementation phased by module. Each module independently deployable behind feature flags. | -| Episodic summaries table grows large | MEDIUM | expires_at column + cleanup cron. Index on expires_at for efficient deletion. | -| Temporal KG backfill slow | LOW | Backfill is optional, runs as cron in background. No downtime. Default valid_from=NOW() sufficient. | -| SQLite compat deferred too long | MEDIUM | Interfaces designed PG-agnostic. SQLite impl can be added later without interface changes. | - ---- - -## Rollback Plan - -### Per-Migration Rollback -```bash -./goclaw migrate down 2 # reverts 000038, then 000037 -``` - -### Per-Feature Rollback -Set feature flag to false in agent settings: -```json -{ "v3_pipeline_enabled": false, "v3_memory_enabled": false } -``` -Agent immediately reverts to v2 behavior. No code rollback needed. - -### Full v3 Rollback -1. Disable all v3 feature flags for all agents -2. Run `migrate down 2` to remove v3 tables -3. Deploy v2 binary -4. v2 memory files, KG entities, sessions — all intact (v3 tables were additive) - ---- - -## Security Considerations - -- **Migration safety:** All changes additive (CREATE TABLE, ALTER ADD COLUMN). No DROP, RENAME, or data modification in up migration. Down migration only drops v3-added objects. -- **Feature flag isolation:** v3 code paths gated by flags. Flag false = v2 code path. No mixed state. -- **Tenant isolation in new tables:** All 3 new tables have tenant_id FK. All queries must include tenant_id filter. Store implementations enforce via context. -- **Evolution metrics cleanup:** Cron job with configurable TTL (default 90 days). Prevents unbounded growth. -- **Temporal KG queries:** Default WHERE valid_until IS NULL produces same results as pre-temporal. No data exposure from temporal columns. - ---- - -## Implementation Dependency Order - -``` -Foundation (can start immediately, no dependencies): - 1. TokenCounter (internal/agent/token_counter.go) - 2. WorkspaceContext (internal/tools/workspace_context.go) - 3. EventBus topics (internal/bus/types.go) - 4. ProviderAdapter interface (internal/providers/adapter.go) - -Core (depends on Foundation): - 5. Pipeline stages (internal/agent/pipeline.go + stage_*.go) - 6. PromptBuilder + templates (internal/agent/prompt_builder.go) - 7. Tool Metadata + enhanced registry (internal/tools/capability.go) - -Memory & KG (depends on Foundation + migrations): - 8. DB migrations 000037 + 000038 - 9. EpisodicSummaryStore (internal/store/episodic_store.go) - 10. KG temporal queries (modify existing pg/ files) - 11. Consolidation worker (internal/agent/consolidation_worker.go) - 12. L0 retriever (internal/agent/retriever.go) - -Integration (depends on Core + Memory): - 13. Hybrid skill searcher (internal/skills/hybrid_searcher.go) - 14. Tool policy capability rules (internal/tools/policy.go) - 15. Provider adapters (all 6 providers) - -Orchestration (depends on Integration): - 16. DelegateTool (internal/tools/delegate_tool.go) - 17. Orchestration mode config (internal/agent/orchestration_mode.go) - 18. Evolution metrics (internal/store/pg/evolution_metrics.go) - 19. Evolution suggestions (internal/agent/evolution_suggestion_engine.go) - -Testing (depends on all above): - 20. Integration tests (tests/integration/v3_*.go) - 21. Regression tests (tests/integration/v2_compat_*.go) -``` - -Items 1-4 are independent and can be built in parallel. -Items 5-7 are independent of each other but depend on 1-4. -Items 8-12 are sequential (migrations first, then stores, then workers). diff --git a/plans/260406-1304-goclaw-v3-core-design/plan.md b/plans/260406-1304-goclaw-v3-core-design/plan.md deleted file mode 100644 index 28617c0d..00000000 --- a/plans/260406-1304-goclaw-v3-core-design/plan.md +++ /dev/null @@ -1,101 +0,0 @@ ---- -title: "GoClaw v3 Core Architecture Design" -description: "Comprehensive design plan for v3 core redesign — 18 modules covering memory, KG, agent loop, providers, workspace, orchestration" -status: complete -priority: P1 -effort: 40h -branch: dev-v3 -tags: [refactor, backend, architecture, design] -blockedBy: [] -blocks: [] -created: 2026-04-06 ---- - -# GoClaw v3 Core Architecture Design - -## Overview - -Design-first plan for comprehensive v3 core redesign. Produces interface contracts, DB schemas, data flow diagrams, and dependency graphs — NOT implementation code. Implementation plans created per-phase after design audit. - -**Brainstorm report:** `plans/reports/brainstorm-260406-1059-goclaw-v3-core-redesign.md` - -## Cross-Plan Dependencies - -None. This plan blocks future implementation plans (to be created after design audit). - -## Key Decisions (from brainstorm) - -| # | Module | Decision | -|---|--------|----------| -| 1 | Loop | Pipeline stages (pluggable interfaces) | -| 2 | Memory | 3-tier (working/episodic/semantic) | -| 3 | Context | L0/L1/L2 progressive loading | -| 4 | KG | Temporal validity windows | -| 5 | Token counting | tiktoken-go accurate | -| 6 | Consolidation | Async event-driven | -| 7 | Retrieval | Smart auto-inject L0 + on-demand | -| 8 | Workspace | Unified WorkspaceContext | -| 9 | System prompt | Template-based | -| 10 | Providers | Capability + unified wire + plugin | -| 11 | Self-evolution | Progressive (metrics→suggest→auto) | -| 12 | Orchestration | Layered: delegate base + team optional | -| 13 | Tools | Redesign (registry, policy, workspace-aware) | -| 14 | Events | Redesign (structured, typed) | -| 15 | Skills | Semantic search + ACL + deps | -| 16 | Context files | DB + better abstraction | -| 17 | Migration | Dual-mode v2→v3 | -| 18 | SQLite | Deferred | - -## Phases - -| Phase | Name | Status | Effort | -|-------|------|--------|--------| -| 1 | [Foundation Interfaces](./phase-01-foundation-interfaces.md) | **Complete** | 8h | -| 2 | [Pipeline Loop Design](./phase-02-pipeline-loop-design.md) | **Complete** | 8h | -| 3 | [Memory & KG Design](./phase-03-memory-kg-design.md) | **Complete** | 8h | -| 4 | [System Integration Design](./phase-04-system-integration-design.md) | **Complete** | 6h | -| 5 | [Orchestration & Intelligence Design](./phase-05-orchestration-intelligence-design.md) | **Complete** | 6h | -| 6 | [Migration & Audit](./phase-06-migration-audit.md) | **Complete** | 4h | - -## Dependency Graph - -``` -Phase 1: Foundation ─────────────────────────┐ - (TokenCounter, WorkspaceContext, │ - EventBus, ProviderAdapter) │ - │ │ - ▼ │ -Phase 2: Pipeline Loop ──────────┐ │ - (Stage interfaces, RunState) │ │ - │ │ │ - ▼ ▼ ▼ -Phase 3: Memory & KG Phase 4: System Integration - (3-tier, temporal, (Template prompt, tool - consolidation, L0/L1/L2) registry, skills, retrieval) - │ │ - └───────────┬───────────┘ - ▼ - Phase 5: Orchestration & Intelligence - (Delegate, self-evolution) - │ - ▼ - Phase 6: Migration & Audit - (v2→v3 dual-mode, test plan) -``` - -## Deliverables Per Phase - -Each phase produces: -- **Interface contracts** — Go interface definitions with method signatures -- **DB schemas** — SQL DDL for new/modified tables -- **Data flow diagrams** — Textual flow descriptions -- **Config schemas** — JSON/YAML configuration structures -- **Dependency map** — Which existing files are affected - -## References - -- Brainstorm: `plans/reports/brainstorm-260406-1059-goclaw-v3-core-redesign.md` -- Research: `plans/reports/researcher-260406-1059-memory-context-kg-research.md` -- Architecture: `plans/reports/Explore-260406-thorough-v3-architecture.md` -- Loop deep dive: `plans/reports/Explore-260406-GoClaw-Agent-Loop-v3-Deep-Dive.md` -- Workspace forcing: `plans/reports/Explore-260406-workspace-forcing.md` diff --git a/tests/integration/vault_classify_test.go b/tests/integration/vault_classify_test.go new file mode 100644 index 00000000..f6d0a5a5 --- /dev/null +++ b/tests/integration/vault_classify_test.go @@ -0,0 +1,578 @@ +//go:build integration + +package integration + +import ( + "database/sql" + "errors" + "testing" + + "github.com/nextlevelbuilder/goclaw/internal/store" + "github.com/nextlevelbuilder/goclaw/internal/store/pg" +) + +func TestVaultClassify_DeleteDocLinksByTypes_SingleType(t *testing.T) { + db := testDB(t) + tenantID, agentID := seedTenantAgent(t, db) + ctx := tenantCtx(tenantID) + vs := newVaultStore(db) + + tid := tenantID.String() + aid := agentID.String() + + // Create 2 documents + docA := makeVaultDoc(tid, aid, "classify/source.md", "Source Doc") + docB := makeVaultDoc(tid, aid, "classify/target.md", "Target Doc") + if err := vs.UpsertDocument(ctx, docA); err != nil { + t.Fatalf("UpsertDocument A: %v", err) + } + if err := vs.UpsertDocument(ctx, docB); err != nil { + t.Fatalf("UpsertDocument B: %v", err) + } + + // Create 3 links with different types + links := []struct { + linkType string + context string + }{ + {"reference", "cited in source"}, + {"wikilink", "mentioned in source"}, + {"depends_on", "dependency of source"}, + } + + for _, l := range links { + link := &store.VaultLink{ + FromDocID: docA.ID, + ToDocID: docB.ID, + LinkType: l.linkType, + Context: l.context, + } + if err := vs.CreateLink(ctx, link); err != nil { + t.Fatalf("CreateLink %s: %v", l.linkType, err) + } + } + + // Verify: 3 links exist + outLinks, err := vs.GetOutLinks(ctx, tid, docA.ID) + if err != nil { + t.Fatalf("GetOutLinks before delete: %v", err) + } + if len(outLinks) != 3 { + t.Errorf("expected 3 links before delete, got %d", len(outLinks)) + } + + // Delete links with type "reference" only + if err := vs.DeleteDocLinksByTypes(ctx, tid, docA.ID, []string{"reference"}); err != nil { + t.Fatalf("DeleteDocLinksByTypes: %v", err) + } + + // Verify: only "reference" deleted, "wikilink" and "depends_on" remain + outLinksAfter, err := vs.GetOutLinks(ctx, tid, docA.ID) + if err != nil { + t.Fatalf("GetOutLinks after delete: %v", err) + } + if len(outLinksAfter) != 2 { + t.Errorf("expected 2 links after deleting 1, got %d", len(outLinksAfter)) + } + + // Verify remaining links are correct types + typeMap := make(map[string]bool) + for _, l := range outLinksAfter { + typeMap[l.LinkType] = true + } + if !typeMap["wikilink"] || !typeMap["depends_on"] { + t.Errorf("expected wikilink and depends_on, got types: %v", typeMap) + } + if typeMap["reference"] { + t.Errorf("reference should have been deleted") + } +} + +func TestVaultClassify_DeleteDocLinksByTypes_MultipleTypes(t *testing.T) { + db := testDB(t) + tenantID, agentID := seedTenantAgent(t, db) + ctx := tenantCtx(tenantID) + vs := newVaultStore(db) + + tid := tenantID.String() + aid := agentID.String() + + // Create source doc and 4 target docs (for 6 different link types + semantic) + docSource := makeVaultDoc(tid, aid, "source.md", "Source") + docs := make([]*store.VaultDocument, 0) + docs = append(docs, docSource) + + for i := 0; i < 7; i++ { + doc := makeVaultDoc(tid, aid, "classify/target-"+string(rune('a'+i))+".md", "Target "+string(rune('A'+i))) + if err := vs.UpsertDocument(ctx, doc); err != nil { + t.Fatalf("UpsertDocument target %d: %v", i, err) + } + docs = append(docs, doc) + } + docSource = docs[0] + + if err := vs.UpsertDocument(ctx, docSource); err != nil { + t.Fatalf("UpsertDocument source: %v", err) + } + + // Create links with 6 classify types + semantic + wikilink (8 total) + linkTypes := []string{ + "reference", // classify type 1 + "depends_on", // classify type 2 + "extends", // classify type 3 + "related", // classify type 4 + "supersedes", // classify type 5 + "contradicts", // classify type 6 + "semantic", // legacy auto-classified type + "wikilink", // manual link type + } + + for i, lt := range linkTypes { + link := &store.VaultLink{ + FromDocID: docSource.ID, + ToDocID: docs[i+1].ID, + LinkType: lt, + Context: "test context for " + lt, + } + if err := vs.CreateLink(ctx, link); err != nil { + t.Fatalf("CreateLink %s: %v", lt, err) + } + } + + // Verify: 8 links created + outLinks, err := vs.GetOutLinks(ctx, tid, docSource.ID) + if err != nil { + t.Fatalf("GetOutLinks before delete: %v", err) + } + if len(outLinks) != 8 { + t.Errorf("expected 8 links before delete, got %d", len(outLinks)) + } + + // Delete all classify types + semantic (7 types), keep wikilink + typesToDelete := []string{ + "reference", "depends_on", "extends", + "related", "supersedes", "contradicts", + "semantic", + } + if err := vs.DeleteDocLinksByTypes(ctx, tid, docSource.ID, typesToDelete); err != nil { + t.Fatalf("DeleteDocLinksByTypes: %v", err) + } + + // Verify: only wikilink survives + outLinksAfter, err := vs.GetOutLinks(ctx, tid, docSource.ID) + if err != nil { + t.Fatalf("GetOutLinks after delete: %v", err) + } + if len(outLinksAfter) != 1 { + t.Errorf("expected 1 link after deleting 7, got %d", len(outLinksAfter)) + } + if outLinksAfter[0].LinkType != "wikilink" { + t.Errorf("expected wikilink, got %s", outLinksAfter[0].LinkType) + } +} + +func TestVaultClassify_DeleteDocLinksByTypes_NoMatches(t *testing.T) { + db := testDB(t) + tenantID, agentID := seedTenantAgent(t, db) + ctx := tenantCtx(tenantID) + vs := newVaultStore(db) + + tid := tenantID.String() + aid := agentID.String() + + // Create docs + docA := makeVaultDoc(tid, aid, "nomatch/a.md", "Doc A") + docB := makeVaultDoc(tid, aid, "nomatch/b.md", "Doc B") + if err := vs.UpsertDocument(ctx, docA); err != nil { + t.Fatalf("UpsertDocument A: %v", err) + } + if err := vs.UpsertDocument(ctx, docB); err != nil { + t.Fatalf("UpsertDocument B: %v", err) + } + + // Create link with type "wikilink" + link := &store.VaultLink{ + FromDocID: docA.ID, + ToDocID: docB.ID, + LinkType: "wikilink", + Context: "manual link", + } + if err := vs.CreateLink(ctx, link); err != nil { + t.Fatalf("CreateLink: %v", err) + } + + // Delete non-existent type — should succeed but delete nothing + if err := vs.DeleteDocLinksByTypes(ctx, tid, docA.ID, []string{"reference"}); err != nil { + t.Fatalf("DeleteDocLinksByTypes non-existent type: %v", err) + } + + // Verify wikilink still exists + outLinks, err := vs.GetOutLinks(ctx, tid, docA.ID) + if err != nil { + t.Fatalf("GetOutLinks after no-op delete: %v", err) + } + if len(outLinks) != 1 { + t.Errorf("expected 1 link (unchanged), got %d", len(outLinks)) + } + if outLinks[0].LinkType != "wikilink" { + t.Errorf("expected wikilink, got %s", outLinks[0].LinkType) + } +} + +func TestVaultClassify_DeleteDocLinksByTypes_EmptyTypeList(t *testing.T) { + db := testDB(t) + tenantID, agentID := seedTenantAgent(t, db) + ctx := tenantCtx(tenantID) + vs := newVaultStore(db) + + tid := tenantID.String() + aid := agentID.String() + + // Create docs and link + docA := makeVaultDoc(tid, aid, "empty/a.md", "Doc A") + docB := makeVaultDoc(tid, aid, "empty/b.md", "Doc B") + if err := vs.UpsertDocument(ctx, docA); err != nil { + t.Fatalf("UpsertDocument A: %v", err) + } + if err := vs.UpsertDocument(ctx, docB); err != nil { + t.Fatalf("UpsertDocument B: %v", err) + } + + link := &store.VaultLink{ + FromDocID: docA.ID, + ToDocID: docB.ID, + LinkType: "reference", + Context: "test", + } + if err := vs.CreateLink(ctx, link); err != nil { + t.Fatalf("CreateLink: %v", err) + } + + // Delete with empty type list — should succeed but delete nothing + if err := vs.DeleteDocLinksByTypes(ctx, tid, docA.ID, []string{}); err != nil { + t.Fatalf("DeleteDocLinksByTypes empty list: %v", err) + } + + // Verify link still exists + outLinks, err := vs.GetOutLinks(ctx, tid, docA.ID) + if err != nil { + t.Fatalf("GetOutLinks after empty delete: %v", err) + } + if len(outLinks) != 1 { + t.Errorf("expected 1 link (unchanged), got %d", len(outLinks)) + } +} + +func TestVaultClassify_DeleteDocLinksByTypes_MultipleSourceDocs(t *testing.T) { + db := testDB(t) + tenantID, agentID := seedTenantAgent(t, db) + ctx := tenantCtx(tenantID) + vs := newVaultStore(db) + + tid := tenantID.String() + aid := agentID.String() + + // Create 2 source docs and 1 target doc + docSource1 := makeVaultDoc(tid, aid, "multi/source1.md", "Source 1") + docSource2 := makeVaultDoc(tid, aid, "multi/source2.md", "Source 2") + docTarget := makeVaultDoc(tid, aid, "multi/target.md", "Target") + + for _, doc := range []*store.VaultDocument{docSource1, docSource2, docTarget} { + if err := vs.UpsertDocument(ctx, doc); err != nil { + t.Fatalf("UpsertDocument: %v", err) + } + } + + // Create links from both sources to target + linkTypes := []string{"reference", "semantic", "wikilink"} + for _, lt := range linkTypes { + link1 := &store.VaultLink{ + FromDocID: docSource1.ID, + ToDocID: docTarget.ID, + LinkType: lt, + Context: "from source1", + } + link2 := &store.VaultLink{ + FromDocID: docSource2.ID, + ToDocID: docTarget.ID, + LinkType: lt, + Context: "from source2", + } + if err := vs.CreateLink(ctx, link1); err != nil { + t.Fatalf("CreateLink source1: %v", err) + } + if err := vs.CreateLink(ctx, link2); err != nil { + t.Fatalf("CreateLink source2: %v", err) + } + } + + // Verify: both sources have 3 links each + outLinks1, err := vs.GetOutLinks(ctx, tid, docSource1.ID) + if err != nil { + t.Fatalf("GetOutLinks source1 before: %v", err) + } + outLinks2, err := vs.GetOutLinks(ctx, tid, docSource2.ID) + if err != nil { + t.Fatalf("GetOutLinks source2 before: %v", err) + } + if len(outLinks1) != 3 || len(outLinks2) != 3 { + t.Errorf("expected 3 links each source before delete, got %d and %d", len(outLinks1), len(outLinks2)) + } + + // Delete "reference" + "semantic" from source1 only + if err := vs.DeleteDocLinksByTypes(ctx, tid, docSource1.ID, []string{"reference", "semantic"}); err != nil { + t.Fatalf("DeleteDocLinksByTypes source1: %v", err) + } + + // Verify: source1 has only "wikilink", source2 unchanged + outLinks1After, err := vs.GetOutLinks(ctx, tid, docSource1.ID) + if err != nil { + t.Fatalf("GetOutLinks source1 after: %v", err) + } + outLinks2After, err := vs.GetOutLinks(ctx, tid, docSource2.ID) + if err != nil { + t.Fatalf("GetOutLinks source2 after: %v", err) + } + + if len(outLinks1After) != 1 { + t.Errorf("expected 1 link in source1 after delete, got %d", len(outLinks1After)) + } + if outLinks1After[0].LinkType != "wikilink" { + t.Errorf("expected wikilink in source1, got %s", outLinks1After[0].LinkType) + } + + if len(outLinks2After) != 3 { + t.Errorf("expected 3 links in source2 (unchanged), got %d", len(outLinks2After)) + } +} + +func TestVaultClassify_DeleteDocLinksByTypes_TenantIsolation(t *testing.T) { + db := testDB(t) + tenantA, tenantB, agentA, agentB := seedTwoTenants(t, db) + ctxA := tenantCtx(tenantA) + ctxB := tenantCtx(tenantB) + vs := newVaultStore(db) + + tidA := tenantA.String() + tidB := tenantB.String() + aidA := agentA.String() + aidB := agentB.String() + + // Create docs in tenant A + docA1 := makeVaultDoc(tidA, aidA, "iso/a1.md", "Tenant A Doc 1") + docA2 := makeVaultDoc(tidA, aidA, "iso/a2.md", "Tenant A Doc 2") + if err := vs.UpsertDocument(ctxA, docA1); err != nil { + t.Fatalf("UpsertDocument tenantA doc1: %v", err) + } + if err := vs.UpsertDocument(ctxA, docA2); err != nil { + t.Fatalf("UpsertDocument tenantA doc2: %v", err) + } + + // Create docs in tenant B + docB1 := makeVaultDoc(tidB, aidB, "iso/b1.md", "Tenant B Doc 1") + docB2 := makeVaultDoc(tidB, aidB, "iso/b2.md", "Tenant B Doc 2") + if err := vs.UpsertDocument(ctxB, docB1); err != nil { + t.Fatalf("UpsertDocument tenantB doc1: %v", err) + } + if err := vs.UpsertDocument(ctxB, docB2); err != nil { + t.Fatalf("UpsertDocument tenantB doc2: %v", err) + } + + // Create links in both tenants + linkA := &store.VaultLink{ + FromDocID: docA1.ID, + ToDocID: docA2.ID, + LinkType: "reference", + Context: "tenant A link", + } + linkB := &store.VaultLink{ + FromDocID: docB1.ID, + ToDocID: docB2.ID, + LinkType: "reference", + Context: "tenant B link", + } + + if err := vs.CreateLink(ctxA, linkA); err != nil { + t.Fatalf("CreateLink tenantA: %v", err) + } + if err := vs.CreateLink(ctxB, linkB); err != nil { + t.Fatalf("CreateLink tenantB: %v", err) + } + + // Verify both have links + outA, err := vs.GetOutLinks(ctxA, tidA, docA1.ID) + if err != nil { + t.Fatalf("GetOutLinks tenantA before: %v", err) + } + outB, err := vs.GetOutLinks(ctxB, tidB, docB1.ID) + if err != nil { + t.Fatalf("GetOutLinks tenantB before: %v", err) + } + if len(outA) != 1 || len(outB) != 1 { + t.Errorf("expected 1 link each before delete, got %d and %d", len(outA), len(outB)) + } + + // Delete from tenant A only — should NOT affect tenant B + if err := vs.DeleteDocLinksByTypes(ctxA, tidA, docA1.ID, []string{"reference"}); err != nil { + t.Fatalf("DeleteDocLinksByTypes tenantA: %v", err) + } + + // Verify: tenantA link deleted, tenantB link unchanged + outAAfter, err := vs.GetOutLinks(ctxA, tidA, docA1.ID) + if err != nil { + t.Fatalf("GetOutLinks tenantA after: %v", err) + } + outBAfter, err := vs.GetOutLinks(ctxB, tidB, docB1.ID) + if err != nil { + t.Fatalf("GetOutLinks tenantB after: %v", err) + } + + if len(outAAfter) != 0 { + t.Errorf("expected 0 links in tenantA after delete, got %d", len(outAAfter)) + } + if len(outBAfter) != 1 { + t.Errorf("expected 1 link in tenantB (unchanged), got %d", len(outBAfter)) + } +} + +func TestVaultClassify_DeleteDocLinksByTypes_NonExistentDoc(t *testing.T) { + db := testDB(t) + tenantID, _ := seedTenantAgent(t, db) + ctx := tenantCtx(tenantID) + vs := newVaultStore(db) + + tid := tenantID.String() + + // Try to delete links from non-existent doc — should succeed (no-op) + fakeDocID := "fake-doc-id-12345" + if err := vs.DeleteDocLinksByTypes(ctx, tid, fakeDocID, []string{"reference"}); err != nil { + t.Fatalf("DeleteDocLinksByTypes on non-existent doc: %v", err) + } + + // GetOutLinks for non-existent doc should return empty or error gracefully + outLinks, err := vs.GetOutLinks(ctx, tid, fakeDocID) + if err != nil && !errors.Is(err, sql.ErrNoRows) { + // Some implementations may error on non-existent doc, others return empty list + // Both are acceptable for a non-existent doc + } + if outLinks != nil && len(outLinks) != 0 { + t.Errorf("expected empty or error for non-existent doc, got %d links", len(outLinks)) + } +} + +func TestVaultClassify_DeleteAllClassifyTypesAndSemantic(t *testing.T) { + db := testDB(t) + tenantID, agentID := seedTenantAgent(t, db) + ctx := tenantCtx(tenantID) + vs := newVaultStore(db) + + tid := tenantID.String() + aid := agentID.String() + + // Create documents + docSource := makeVaultDoc(tid, aid, "classify/source.md", "Source Doc") + docTarget1 := makeVaultDoc(tid, aid, "classify/target1.md", "Target 1") + docTarget2 := makeVaultDoc(tid, aid, "classify/target2.md", "Target 2") + + for _, doc := range []*store.VaultDocument{docSource, docTarget1, docTarget2} { + if err := vs.UpsertDocument(ctx, doc); err != nil { + t.Fatalf("UpsertDocument: %v", err) + } + } + + // Create links with mix of classify types, semantic, and manual types + linkConfigs := []struct { + toDocID string + linkType string + }{ + {docTarget1.ID, "reference"}, + {docTarget1.ID, "semantic"}, + {docTarget2.ID, "depends_on"}, + {docTarget2.ID, "wikilink"}, + } + + for _, cfg := range linkConfigs { + link := &store.VaultLink{ + FromDocID: docSource.ID, + ToDocID: cfg.toDocID, + LinkType: cfg.linkType, + Context: "test context", + } + if err := vs.CreateLink(ctx, link); err != nil { + t.Fatalf("CreateLink %s: %v", cfg.linkType, err) + } + } + + // Verify: 4 links created + outLinks, err := vs.GetOutLinks(ctx, tid, docSource.ID) + if err != nil { + t.Fatalf("GetOutLinks before delete: %v", err) + } + if len(outLinks) != 4 { + t.Errorf("expected 4 links before delete, got %d", len(outLinks)) + } + + // Delete all classify types (reference, depends_on) + semantic, keep wikilink + if err := vs.DeleteDocLinksByTypes(ctx, tid, docSource.ID, []string{ + "reference", "depends_on", "extends", "related", "supersedes", "contradicts", "semantic", + }); err != nil { + t.Fatalf("DeleteDocLinksByTypes: %v", err) + } + + // Verify: only wikilink remains + outLinksAfter, err := vs.GetOutLinks(ctx, tid, docSource.ID) + if err != nil { + t.Fatalf("GetOutLinks after delete: %v", err) + } + if len(outLinksAfter) != 1 { + t.Errorf("expected 1 link after cleanup, got %d", len(outLinksAfter)) + } + if outLinksAfter[0].LinkType != "wikilink" { + t.Errorf("expected wikilink survivor, got %s", outLinksAfter[0].LinkType) + } +} + +func TestVaultClassify_DeleteDocLinksByTypes_CasePreservation(t *testing.T) { + db := testDB(t) + tenantID, agentID := seedTenantAgent(t, db) + ctx := tenantCtx(tenantID) + vs := newVaultStore(db) + + tid := tenantID.String() + aid := agentID.String() + + // Create documents + docA := makeVaultDoc(tid, aid, "case/a.md", "Doc A") + docB := makeVaultDoc(tid, aid, "case/b.md", "Doc B") + if err := vs.UpsertDocument(ctx, docA); err != nil { + t.Fatalf("UpsertDocument A: %v", err) + } + if err := vs.UpsertDocument(ctx, docB); err != nil { + t.Fatalf("UpsertDocument B: %v", err) + } + + // Create links with different case sensitivity + link := &store.VaultLink{ + FromDocID: docA.ID, + ToDocID: docB.ID, + LinkType: "depends_on", // lowercase with underscore + Context: "test", + } + if err := vs.CreateLink(ctx, link); err != nil { + t.Fatalf("CreateLink: %v", err) + } + + // Try deleting with exact case match + if err := vs.DeleteDocLinksByTypes(ctx, tid, docA.ID, []string{"depends_on"}); err != nil { + t.Fatalf("DeleteDocLinksByTypes: %v", err) + } + + // Verify link is deleted + outLinks, err := vs.GetOutLinks(ctx, tid, docA.ID) + if err != nil { + t.Fatalf("GetOutLinks: %v", err) + } + if len(outLinks) != 0 { + t.Errorf("expected 0 links after exact case match delete, got %d", len(outLinks)) + } +} diff --git a/ui/desktop/frontend/src/components/agents/evolution-suggestions-list.tsx b/ui/desktop/frontend/src/components/agents/evolution-suggestions-list.tsx index 331435bd..7d25c7e0 100644 --- a/ui/desktop/frontend/src/components/agents/evolution-suggestions-list.tsx +++ b/ui/desktop/frontend/src/components/agents/evolution-suggestions-list.tsx @@ -14,7 +14,6 @@ const TYPE_COLORS: Record = { threshold: 'bg-blue-500/10 text-blue-600', tool_order: 'bg-orange-500/10 text-orange-600', skill_add: 'bg-green-500/10 text-green-600', - memory_prune: 'bg-surface-tertiary text-text-muted', } const STATUS_COLORS: Record = { diff --git a/ui/web/package.json b/ui/web/package.json index e78ae991..0b287681 100644 --- a/ui/web/package.json +++ b/ui/web/package.json @@ -14,14 +14,19 @@ "@dnd-kit/sortable": "^10.0.0", "@dnd-kit/utilities": "^3.2.2", "@hookform/resolvers": "^5.2.2", + "@sigma/edge-curve": "^3.1.0", "@tailwindcss/typography": "^0.5.19", "@tanstack/react-query": "^5.95.2", "@tanstack/react-table": "^8.21.3", "class-variance-authority": "^0.7.1", "clsx": "^2.1.1", - "d3-force": "^3.0.0", "date-fns": "^4.1.0", "framer-motion": "^12.38.0", + "graphology": "^0.26.0", + "graphology-layout": "^0.6.1", + "graphology-layout-forceatlas2": "^0.10.1", + "graphology-layout-noverlap": "^0.4.2", + "graphology-types": "^0.24.8", "highlight.js": "^11.11.1", "i18next": "^26.0.1", "jszip": "^3.10.1", @@ -30,7 +35,6 @@ "radix-ui": "^1.4.3", "react": "^19.2.4", "react-dom": "^19.2.4", - "react-force-graph-2d": "^1.29.1", "react-hook-form": "^7.72.1", "react-i18next": "^17.0.1", "react-markdown": "^10.1.0", @@ -40,6 +44,7 @@ "rehype-katex": "^7.0.1", "remark-gfm": "^4.0.1", "remark-math": "^6.0.0", + "sigma": "^3.0.2", "tailwind-merge": "^3.5.0", "unist-util-visit": "^5.1.0", "zod": "^4.3.6", @@ -47,7 +52,6 @@ }, "devDependencies": { "@tailwindcss/vite": "^4.2.2", - "@types/d3-force": "^3.0.10", "@types/mdast": "^4.0.4", "@types/react": "^19.2.14", "@types/react-dom": "^19.2.3", diff --git a/ui/web/pnpm-lock.yaml b/ui/web/pnpm-lock.yaml index 6668d561..aceed6cf 100644 --- a/ui/web/pnpm-lock.yaml +++ b/ui/web/pnpm-lock.yaml @@ -20,6 +20,9 @@ importers: '@hookform/resolvers': specifier: ^5.2.2 version: 5.2.2(react-hook-form@7.72.1(react@19.2.4)) + '@sigma/edge-curve': + specifier: ^3.1.0 + version: 3.1.0(sigma@3.0.2(graphology-types@0.24.8)) '@tailwindcss/typography': specifier: ^0.5.19 version: 0.5.19(tailwindcss@4.2.2) @@ -35,15 +38,27 @@ importers: clsx: specifier: ^2.1.1 version: 2.1.1 - d3-force: - specifier: ^3.0.0 - version: 3.0.0 date-fns: specifier: ^4.1.0 version: 4.1.0 framer-motion: specifier: ^12.38.0 version: 12.38.0(react-dom@19.2.4(react@19.2.4))(react@19.2.4) + graphology: + specifier: ^0.26.0 + version: 0.26.0(graphology-types@0.24.8) + graphology-layout: + specifier: ^0.6.1 + version: 0.6.1(graphology-types@0.24.8) + graphology-layout-forceatlas2: + specifier: ^0.10.1 + version: 0.10.1(graphology-types@0.24.8) + graphology-layout-noverlap: + specifier: ^0.4.2 + version: 0.4.2(graphology-types@0.24.8) + graphology-types: + specifier: ^0.24.8 + version: 0.24.8 highlight.js: specifier: ^11.11.1 version: 11.11.1 @@ -68,9 +83,6 @@ importers: react-dom: specifier: ^19.2.4 version: 19.2.4(react@19.2.4) - react-force-graph-2d: - specifier: ^1.29.1 - version: 1.29.1(react@19.2.4) react-hook-form: specifier: ^7.72.1 version: 7.72.1(react@19.2.4) @@ -98,6 +110,9 @@ importers: remark-math: specifier: ^6.0.0 version: 6.0.0 + sigma: + specifier: ^3.0.2 + version: 3.0.2(graphology-types@0.24.8) tailwind-merge: specifier: ^3.5.0 version: 3.5.0 @@ -114,9 +129,6 @@ importers: '@tailwindcss/vite': specifier: ^4.2.2 version: 4.2.2(vite@8.0.3(@emnapi/core@1.9.1)(@emnapi/runtime@1.9.1)(@types/node@25.5.0)(jiti@2.6.1)) - '@types/d3-force': - specifier: ^3.0.10 - version: 3.0.10 '@types/mdast': specifier: ^4.0.4 version: 4.0.4 @@ -1062,6 +1074,11 @@ packages: '@rolldown/pluginutils@1.0.0-rc.7': resolution: {integrity: sha512-qujRfC8sFVInYSPPMLQByRh7zhwkGFS4+tyMQ83srV1qrxL4g8E2tyxVVyxd0+8QeBM1mIk9KbWxkegRr76XzA==} + '@sigma/edge-curve@3.1.0': + resolution: {integrity: sha512-OFWkfAXEsm+X8l1K4K49cC0psB0gQ+gqxKA08HG5piNPdzrDZ5gG9Gza6htZ5AirOVwd/4/uq/gPpD5En+H+3Q==} + peerDependencies: + sigma: '>=3.0.0-beta.10' + '@standard-schema/spec@1.1.0': resolution: {integrity: sha512-l2aFy5jALhniG5HgqrD6jXLi/rUWrKvqN/qJx6yoJsgKhblVd+iqqU4RCXavm/jPityDo5TCvKMnpjKnOriy0w==} @@ -1186,9 +1203,6 @@ packages: resolution: {integrity: sha512-ldZXEhOBb8Is7xLs01fR3YEc3DERiz5silj8tnGkFZytt1abEvl/GhUmCE0PMLaMPTa3Jk4HbKmRlHmu+gCftg==} engines: {node: '>=12'} - '@tweenjs/tween.js@25.0.0': - resolution: {integrity: sha512-XKLA6syeBUaPzx4j3qwMqzzq+V4uo72BnlbOjmuljLrRqdsd3qnzvZZoxvMHZ23ndsRS4aufU6JOZYpCbU6T1A==} - '@tybys/wasm-util@0.10.1': resolution: {integrity: sha512-9tTaPJLSiejZKx+Bmog4uSubteqTvFrVrURwkmHixBo0G4seD0zUxp98E1DzUBJxLQ3NPwXrGKDiVjwx/DpPsg==} @@ -1354,10 +1368,6 @@ packages: babel-plugin-react-compiler: optional: true - accessor-fn@1.5.3: - resolution: {integrity: sha512-rkAofCwe/FvYFUlMB0v0gWmhqtfAtV1IUkdPbfhTUyYniu5LrC0A0UJkTH0Jv3S8SvwkmfuAlY+mQIJATdocMA==} - engines: {node: '>=12'} - acorn@8.16.0: resolution: {integrity: sha512-UVJyE9MttOsBQIDKw1skb9nAwQuR5wuGD3+82K6JgJlm/Y+KI92oNsMNGZCYdDsVtRHSak0pcV5Dno5+4jh9sw==} engines: {node: '>=0.4.0'} @@ -1370,13 +1380,6 @@ packages: bail@2.0.2: resolution: {integrity: sha512-0xO6mYd7JB2YesxDKplafRpsiOzPt9V02ddPCLbY1xYGPOX24NTyN50qnUxgCPcSoYMhKpAuBTjQoRZCAkUDRw==} - bezier-js@6.1.4: - resolution: {integrity: sha512-PA0FW9ZpcHbojUCMu28z9Vg/fNkwTj5YhusSAjHHDfHDGLxJ6YUKrAN2vk1fP2MMOxVw4Oko16FMlRGVBGqLKg==} - - canvas-color-tracker@1.3.2: - resolution: {integrity: sha512-ryQkDX26yJ3CXzb3hxUVNlg1NKE4REc5crLBq661Nxzr8TNd236SaEf2ffYLXyI5tSABSeguHLqcVq4vf9L3Zg==} - engines: {node: '>=12'} - ccount@2.0.1: resolution: {integrity: sha512-eyrF0jiFpY+3drT6383f1qhkbGsLSifNAjA61IUjZjmLCWjItY6LB9ft9YhoDgwfmclB2zhu51Lc7+95b8NRAg==} @@ -1468,9 +1471,6 @@ packages: resolution: {integrity: sha512-IH5tgjV4jE/GhHkRV0HiVYPDtvfjHQlQfJHs0usq7M30XcSBvOotpmH1IgkcXsO/5gEQZD43B//fc7SRT5S+xw==} engines: {node: '>=12'} - d3-binarytree@1.0.2: - resolution: {integrity: sha512-cElUNH+sHu95L04m92pG73t2MEJXKu+GeKUN1TJkFsu93E5W8E9Sc3kHEGJKgenGvj19m6upSn2EunvMgMD2Yw==} - d3-brush@3.0.0: resolution: {integrity: sha512-ALnjWlVYkXsVIGlOsuWH1+3udkYFI48Ljihfnh8FZPF2QS9o+PzGLBslO0PjzVoHLZ2KCVgAM8NVkXPJB2aNnQ==} engines: {node: '>=12'} @@ -1512,10 +1512,6 @@ packages: resolution: {integrity: sha512-kpkQIM20n3oLVBKGg6oHrUchHM3xODkTzjMoj7aWQFq5QEM+R6E4WkzT5+tojDY7yjez8KgCBRoj4aEr99Fdqw==} engines: {node: '>=12'} - d3-force-3d@3.0.6: - resolution: {integrity: sha512-4tsKHUPLOVkyfEffZo1v6sFHvGFwAIIjt/W8IThbp08DYAsXZck+2pSHEG5W1+gQgEvFLdZkYvmJAbRM2EzMnA==} - engines: {node: '>=12'} - d3-force@3.0.0: resolution: {integrity: sha512-zxV/SsA+U4yte8051P4ECydjD/S+qeYtnaIyAs9tgHCqfguma/aAQDjo85A9Z6EKhBirHRJHXIgJUlffT4wdLg==} engines: {node: '>=12'} @@ -1536,9 +1532,6 @@ packages: resolution: {integrity: sha512-3bYs1rOD33uo8aqJfKP3JWPAibgw8Zm2+L9vBKEHJ2Rg+viTR7o5Mmv5mZcieN+FRYaAOWX5SJATX6k1PWz72g==} engines: {node: '>=12'} - d3-octree@1.1.0: - resolution: {integrity: sha512-F8gPlqpP+HwRPMO/8uOu5wjH110+6q4cgJvgJT6vlpy3BEaDIKlTZrgHKZSp/i1InRpVfh4puY/kvL6MxK930A==} - d3-path@1.0.9: resolution: {integrity: sha512-VLaYcn81dtHVTjEHd8B+pbe9yHWpXKZUC87PzoFmsFrJqgFwDe/qxfp5MlfsfM1V5E/iVt0MmEbWQ7FVIXh/bg==} @@ -1671,6 +1664,10 @@ packages: eventemitter3@5.0.4: resolution: {integrity: sha512-mlsTRyGaPBjPedk6Bvw+aqbsXDtoAyAzm5MO7JgU+yVRyMQ5O8bD4Kcci7BS85f93veegeCPkL8R4GLClnjLFw==} + events@3.3.0: + resolution: {integrity: sha512-mQw+2fkQbALzQ7V0MY0IqdnXNOeTtP4r0lN9z7AAawCXgqea7bDii20AYrIBrFd/Hx0M2Ocz6S111CaFkUcb0Q==} + engines: {node: '>=0.8.x'} + extend@3.0.2: resolution: {integrity: sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g==} @@ -1683,14 +1680,6 @@ packages: picomatch: optional: true - float-tooltip@1.7.5: - resolution: {integrity: sha512-/kXzuDnnBqyyWyhDMH7+PfP8J/oXiAavGzcRxASOMRHFuReDtofizLLJsf7nnDLAfEaMW4pVWaXrAjtnglpEkg==} - engines: {node: '>=12'} - - force-graph@1.51.2: - resolution: {integrity: sha512-zZNdMqx8qIQGurgnbgYIUsdXxSfvhfRSIdncsKGv/twUOZpwCsk9hPHmdjdcme1+epATgb41G0rkIGHJ0Wydng==} - engines: {node: '>=12'} - framer-motion@12.38.0: resolution: {integrity: sha512-rFYkY/pigbcswl1XQSb7q424kSTQ8q6eAC+YUsSKooHQYuLdzdHjrt6uxUC+PRAO++q5IS7+TamgIw1AphxR+g==} peerDependencies: @@ -1717,6 +1706,34 @@ packages: graceful-fs@4.2.11: resolution: {integrity: sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==} + graphology-layout-forceatlas2@0.10.1: + resolution: {integrity: sha512-ogzBeF1FvWzjkikrIFwxhlZXvD2+wlY54lqhsrWprcdPjopM2J9HoMweUmIgwaTvY4bUYVimpSsOdvDv1gPRFQ==} + peerDependencies: + graphology-types: '>=0.19.0' + + graphology-layout-noverlap@0.4.2: + resolution: {integrity: sha512-13WwZSx96zim6l1dfZONcqLh3oqyRcjIBsqz2c2iJ3ohgs3605IDWjldH41Gnhh462xGB1j6VGmuGhZ2FKISXA==} + peerDependencies: + graphology-types: '>=0.19.0' + + graphology-layout@0.6.1: + resolution: {integrity: sha512-m9aMvbd0uDPffUCFPng5ibRkb2pmfNvdKjQWeZrf71RS1aOoat5874+DcyNfMeCT4aQguKC7Lj9eCbqZj/h8Ag==} + peerDependencies: + graphology-types: '>=0.19.0' + + graphology-types@0.24.8: + resolution: {integrity: sha512-hDRKYXa8TsoZHjgEaysSRyPdT6uB78Ci8WnjgbStlQysz7xR52PInxNsmnB7IBOM1BhikxkNyCVEFgmPKnpx3Q==} + + graphology-utils@2.5.2: + resolution: {integrity: sha512-ckHg8MXrXJkOARk56ZaSCM1g1Wihe2d6iTmz1enGOz4W/l831MBCKSayeFQfowgF8wd+PQ4rlch/56Vs/VZLDQ==} + peerDependencies: + graphology-types: '>=0.23.0' + + graphology@0.26.0: + resolution: {integrity: sha512-8SSImzgUUYC89Z042s+0r/vMibY7GX/Emz4LDO5e7jYXhuoWfHISPFJYjpRLUSJGq6UQ6xlenvX1p/hJdfXuXg==} + peerDependencies: + graphology-types: '>=0.24.0' + hachure-fill@0.5.2: resolution: {integrity: sha512-3GKBOn+m2LX9iq+JC1064cSFprJY4jL1jCXTcpnfER5HYE2l/4EfWSGzkPa/ZDBmYI0ZOEj5VHV/eKnPGkHuOg==} @@ -1781,10 +1798,6 @@ packages: immer@11.1.4: resolution: {integrity: sha512-XREFCPo6ksxVzP4E0ekD5aMdf8WMwmdNaz6vuvxgI40UaEiu6q3p8X52aU6GdyvLY3XXX/8R7JOTXStz/nBbRw==} - index-array-by@1.4.2: - resolution: {integrity: sha512-SP23P27OUKzXWEC/TOyWlwLviofQkCSCKONnc62eItjp69yCZZPqDQtr3Pw5gJDnPeUMqExmKydNZaJO0FU9pw==} - engines: {node: '>=12'} - inherits@2.0.4: resolution: {integrity: sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==} @@ -1817,24 +1830,13 @@ packages: isarray@1.0.0: resolution: {integrity: sha512-VLghIWNM6ELQzo7zwmcg0NmTVyWKYjvIeM83yjp0wRDTmUnrM678fQbcKBo6n2CJEF0szoG//ytg+TKla89ALQ==} - jerrypick@1.1.2: - resolution: {integrity: sha512-YKnxXEekXKzhpf7CLYA0A+oDP8V0OhICNCr5lv96FvSsDEmrb0GKM776JgQvHTMjr7DTTPEVv/1Ciaw0uEWzBA==} - engines: {node: '>=12'} - jiti@2.6.1: resolution: {integrity: sha512-ekilCSN1jwRvIbgeg/57YFh8qQDNbwDb9xT/qu2DAHbFFZUicIl4ygVaAvzveMhMVr3LnpSKTNnwt8PoOfmKhQ==} hasBin: true - js-tokens@4.0.0: - resolution: {integrity: sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==} - jszip@3.10.1: resolution: {integrity: sha512-xXDvecyTpGLrqFrvkrUSoxxfJI5AH7U8zxxtVclpsUtMCq4JQ290LY8AW5c7Ggnr/Y/oK+bQMbqK2qmtk3pN4g==} - kapsule@1.16.3: - resolution: {integrity: sha512-4+5mNNf4vZDSwPhKprKwz3330iisPrb08JyMgbsdFrimBCKNHecua/WBwvVg3n7vwx0C1ARjfhwIpbrbd9n5wg==} - engines: {node: '>=12'} - katex@0.16.45: resolution: {integrity: sha512-pQpZbdBu7wCTmQUh7ufPmLr0pFoObnGUoL/yhtwJDgmmQpbkg/0HSVti25Fu4rmd1oCR6NGWe9vqTWuWv3GcNA==} hasBin: true @@ -1935,10 +1937,6 @@ packages: longest-streak@3.1.0: resolution: {integrity: sha512-9Ri+o0JYgehTaVBBDoMqIl8GXtbWg711O3srftcHhZ0dqnETqLaoIK0x17fUw9rFSlK/0NlsKe0Ahhyl5pXE2g==} - loose-envify@1.4.0: - resolution: {integrity: sha512-lyuxPGr/Wfhrlem2CL/UcnUc1zcqKAImBDzukY7Y5F/yQiNdko6+fRLevlw1HgMySw7f611UIY408EtxRSoK3Q==} - hasBin: true - lowlight@3.3.0: resolution: {integrity: sha512-0JNhgFoPvP6U6lE/UdVsSq99tn6DhjjpAj5MxG49ewd2mOBVtwWYIT8ClyABhq198aXXODMU6Ox8DrGy/CpTZQ==} @@ -2099,6 +2097,9 @@ packages: mlly@1.8.2: resolution: {integrity: sha512-d+ObxMQFmbt10sretNDytwt85VrbkhhUA/JBGm1MPaWJ65Cl4wOgLaB1NYvJSZ0Ef03MMEU/0xpPMXUIQ29UfA==} + mnemonist@0.39.8: + resolution: {integrity: sha512-vyWo2K3fjrUw8YeeZ1zF0fy6Mu59RHokURlld8ymdUPjMlD9EC9ov1/YPqTgqRvUN9nTr3Gqfz29LYAmu0PHPQ==} + motion-dom@12.38.0: resolution: {integrity: sha512-pdkHLD8QYRp8VfiNLb8xIBJis1byQ9gPT3Jnh2jqfFtAsWUA3dEepDlsWe/xMpO8McV+VdpKVcp+E+TGJEtOoA==} @@ -2113,9 +2114,8 @@ packages: engines: {node: ^10 || ^12 || ^13.7 || ^14 || >=15.0.1} hasBin: true - object-assign@4.1.1: - resolution: {integrity: sha512-rJgTQnkUnH1sFw8yT6VSU3zD3sWmu6sZhIseY8VX+GRu3P6F7Fu+JNDoXfklElbLJSnc3FUQHVe4cU5hj+BcUg==} - engines: {node: '>=0.10.0'} + obliterator@2.0.5: + resolution: {integrity: sha512-42CPE9AhahZRsMNslczq0ctAEtqk8Eka26QofnqC346BZdHDySk3LWka23LI7ULIw11NmltpiLagIq8gBozxTw==} package-manager-detector@1.6.0: resolution: {integrity: sha512-61A5ThoTiDG/C8s8UMZwSorAGwMJ0ERVGj2OjoW5pAalsNOg15+iQiPzrLJ4jhZ1HJzmC2PIHT2oEiH3R5fzNA==} @@ -2123,6 +2123,9 @@ packages: pako@1.0.11: resolution: {integrity: sha512-4hLB8Py4zZce5s4yd9XzopqwVv/yGNhV1Bl8NTmCq1763HeK2+EwVTv+leGeL13Dnh2wfbqowVPXCIO0z4taYw==} + pandemonium@2.4.1: + resolution: {integrity: sha512-wRqjisUyiUfXowgm7MFH2rwJzKIr20rca5FsHXCMNm1W5YPP1hCtrZfgmQ62kP7OZ7Xt+cR858aB28lu5NX55g==} + parse-entities@4.0.2: resolution: {integrity: sha512-GG2AQYWoLgL877gQIKeRPGO1xF9+eG1ujIb5soS5gPvLQ1y2o8FL90w2QWNdf9I361Mpp7726c+lj3U0qK1uGw==} @@ -2159,15 +2162,9 @@ packages: resolution: {integrity: sha512-OW/rX8O/jXnm82Ey1k44pObPtdblfiuWnrd8X7GJ7emImCOstunGbXUpp7HdBrFQX6rJzn3sPT397Wp5aCwCHg==} engines: {node: ^10 || ^12 || >=14} - preact@10.29.1: - resolution: {integrity: sha512-gQCLc/vWroE8lIpleXtdJhTFDogTdZG9AjMUpVkDf2iTCNwYNWA+u16dL41TqUDJO4gm2IgrcMv3uTpjd4Pwmg==} - process-nextick-args@2.0.1: resolution: {integrity: sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==} - prop-types@15.8.1: - resolution: {integrity: sha512-oj87CgZICdulUohogVAR7AjlC0327U4el4L6eAvOqCeudMDVU0NThNaV+b9Df4dXgSP1gXMTnPdhfe/2qDH5cg==} - property-information@7.1.0: resolution: {integrity: sha512-TwEZ+X+yCJmYfL7TPUOcvBZ4QfoT5YenQiJuX//0th53DE6w0xxLEtfK3iyryQFddXuvkIk51EEgrJQ0WJkOmQ==} @@ -2189,12 +2186,6 @@ packages: peerDependencies: react: ^19.2.4 - react-force-graph-2d@1.29.1: - resolution: {integrity: sha512-1Rl/1Z3xy2iTHKj6a0jRXGyiI86xUti81K+jBQZ+Oe46csaMikp47L5AjrzA9hY9fNGD63X8ffrqnvaORukCuQ==} - engines: {node: '>=12'} - peerDependencies: - react: '*' - react-hook-form@7.72.1: resolution: {integrity: sha512-RhwBoy2ygeVZje+C+bwJ8g0NjTdBmDlJvAUHTxRjTmSUKPYsKfMphkS2sgEMotsY03bP358yEYlnUeZy//D9Ig==} engines: {node: '>=18.0.0'} @@ -2217,18 +2208,9 @@ packages: typescript: optional: true - react-is@16.13.1: - resolution: {integrity: sha512-24e6ynE2H+OKt4kqsOvNd8kBpV65zoxbA4BVsEOB3ARVWQki/DHzaUoC5KuON/BiccDaCCTZBuOcfZs70kR8bQ==} - react-is@19.2.4: resolution: {integrity: sha512-W+EWGn2v0ApPKgKKCy/7s7WHXkboGcsrXE+2joLyVxkbyVQfO3MUEaUQDHoSmb8TFFrSKYa9mw64WZHNHSDzYA==} - react-kapsule@2.5.7: - resolution: {integrity: sha512-kifAF4ZPD77qZKc4CKLmozq6GY1sBzPEJTIJb0wWFK6HsePJatK3jXplZn2eeAt3x67CDozgi7/rO8fNQ/AL7A==} - engines: {node: '>=12'} - peerDependencies: - react: '>=16.13.1' - react-markdown@10.1.0: resolution: {integrity: sha512-qKxVopLT/TyA6BX3Ue5NwabOsAzm0Q7kAPwq6L+wWDwisYs7R8vZ0nRXqq6rkueboxpkjvLGU9fWifiX/ZZFxQ==} peerDependencies: @@ -2363,6 +2345,9 @@ packages: setimmediate@1.0.5: resolution: {integrity: sha512-MATJdZp8sLqDl/68LfQmbP8zKPLQNV6BIZoIgrscFDQ+RsvK/BxeDQOgyxKKoh0y/8h3BqVFnCqQ/gd+reiIXA==} + sigma@3.0.2: + resolution: {integrity: sha512-/BUbeOwPGruiBOm0YQQ6ZMcLIZ6tf/W+Jcm7dxZyAX0tK3WP9/sq7/NAWBxPIxVahdGjCJoGwej0Gdrv0DxlQQ==} + source-map-js@1.2.1: resolution: {integrity: sha512-UXWMKhLOwVKb728IUtQPXxfYU+usdybtUrK/8uGE8CQMvrhOpwvzDBwj0QhSL7MQc7vIsISBG8VQ8+IDQxpfQA==} engines: {node: '>=0.10.0'} @@ -2398,9 +2383,6 @@ packages: tiny-invariant@1.3.3: resolution: {integrity: sha512-+FbBPE1o9QAYvviau/qC5SE3caw21q3xkvWKBtja5vgqOWIHHJ3ioaq1VPfn/Szqctz2bU/oYeKd9/z5BL+PVg==} - tinycolor2@1.6.0: - resolution: {integrity: sha512-XPaBkWQJdsf3pLKJV9p4qN/S+fm2Oj8AIPo1BTUhg5oxkvm9+SVEGFdhyOz7tTdUTfvxMiAs4sp6/eZO2Ew+pw==} - tinyexec@1.1.1: resolution: {integrity: sha512-VKS/ZaQhhkKFMANmAOhhXVoIfBXblQxGX1myCQ2faQrfmobMftXeJPcZGp0gS07ocvGJWDLZGyOZDadDBqYIJg==} engines: {node: '>=18'} @@ -3549,6 +3531,10 @@ snapshots: '@rolldown/pluginutils@1.0.0-rc.7': {} + '@sigma/edge-curve@3.1.0(sigma@3.0.2(graphology-types@0.24.8))': + dependencies: + sigma: 3.0.2(graphology-types@0.24.8) + '@standard-schema/spec@1.1.0': {} '@standard-schema/utils@0.3.0': {} @@ -3641,8 +3627,6 @@ snapshots: '@tanstack/table-core@8.21.3': {} - '@tweenjs/tween.js@25.0.0': {} - '@tybys/wasm-util@0.10.1': dependencies: tslib: 2.8.1 @@ -3826,8 +3810,6 @@ snapshots: '@rolldown/pluginutils': 1.0.0-rc.7 vite: 8.0.3(@emnapi/core@1.9.1)(@emnapi/runtime@1.9.1)(@types/node@25.5.0)(jiti@2.6.1) - accessor-fn@1.5.3: {} - acorn@8.16.0: {} aria-hidden@1.2.6: @@ -3836,12 +3818,6 @@ snapshots: bail@2.0.2: {} - bezier-js@6.1.4: {} - - canvas-color-tracker@1.3.2: - dependencies: - tinycolor2: 1.6.0 - ccount@2.0.1: {} character-entities-html4@2.1.0: {} @@ -3917,8 +3893,6 @@ snapshots: d3-axis@3.0.0: {} - d3-binarytree@1.0.2: {} - d3-brush@3.0.0: dependencies: d3-dispatch: 3.0.1 @@ -3960,14 +3934,6 @@ snapshots: dependencies: d3-dsv: 3.0.1 - d3-force-3d@3.0.6: - dependencies: - d3-binarytree: 1.0.2 - d3-dispatch: 3.0.1 - d3-octree: 1.1.0 - d3-quadtree: 3.0.1 - d3-timer: 3.0.1 - d3-force@3.0.0: dependencies: d3-dispatch: 3.0.1 @@ -3986,8 +3952,6 @@ snapshots: dependencies: d3-color: 3.1.0 - d3-octree@1.1.0: {} - d3-path@1.0.9: {} d3-path@3.1.0: {} @@ -4138,36 +4102,14 @@ snapshots: eventemitter3@5.0.4: {} + events@3.3.0: {} + extend@3.0.2: {} fdir@6.5.0(picomatch@4.0.4): optionalDependencies: picomatch: 4.0.4 - float-tooltip@1.7.5: - dependencies: - d3-selection: 3.0.0 - kapsule: 1.16.3 - preact: 10.29.1 - - force-graph@1.51.2: - dependencies: - '@tweenjs/tween.js': 25.0.0 - accessor-fn: 1.5.3 - bezier-js: 6.1.4 - canvas-color-tracker: 1.3.2 - d3-array: 3.2.4 - d3-drag: 3.0.0 - d3-force-3d: 3.0.6 - d3-scale: 4.0.2 - d3-scale-chromatic: 3.1.0 - d3-selection: 3.0.0 - d3-zoom: 3.0.0 - float-tooltip: 1.7.5 - index-array-by: 1.4.2 - kapsule: 1.16.3 - lodash-es: 4.18.1 - framer-motion@12.38.0(react-dom@19.2.4(react@19.2.4))(react@19.2.4): dependencies: motion-dom: 12.38.0 @@ -4184,6 +4126,33 @@ snapshots: graceful-fs@4.2.11: {} + graphology-layout-forceatlas2@0.10.1(graphology-types@0.24.8): + dependencies: + graphology-types: 0.24.8 + graphology-utils: 2.5.2(graphology-types@0.24.8) + + graphology-layout-noverlap@0.4.2(graphology-types@0.24.8): + dependencies: + graphology-types: 0.24.8 + graphology-utils: 2.5.2(graphology-types@0.24.8) + + graphology-layout@0.6.1(graphology-types@0.24.8): + dependencies: + graphology-types: 0.24.8 + graphology-utils: 2.5.2(graphology-types@0.24.8) + pandemonium: 2.4.1 + + graphology-types@0.24.8: {} + + graphology-utils@2.5.2(graphology-types@0.24.8): + dependencies: + graphology-types: 0.24.8 + + graphology@0.26.0(graphology-types@0.24.8): + dependencies: + events: 3.3.0 + graphology-types: 0.24.8 + hachure-fill@0.5.2: {} hast-util-from-dom@5.0.1: @@ -4290,8 +4259,6 @@ snapshots: immer@11.1.4: {} - index-array-by@1.4.2: {} - inherits@2.0.4: {} inline-style-parser@0.2.7: {} @@ -4315,12 +4282,8 @@ snapshots: isarray@1.0.0: {} - jerrypick@1.1.2: {} - jiti@2.6.1: {} - js-tokens@4.0.0: {} - jszip@3.10.1: dependencies: lie: 3.3.0 @@ -4328,10 +4291,6 @@ snapshots: readable-stream: 2.3.8 setimmediate: 1.0.5 - kapsule@1.16.3: - dependencies: - lodash-es: 4.18.1 - katex@0.16.45: dependencies: commander: 8.3.0 @@ -4408,10 +4367,6 @@ snapshots: longest-streak@3.1.0: {} - loose-envify@1.4.0: - dependencies: - js-tokens: 4.0.0 - lowlight@3.3.0: dependencies: '@types/hast': 3.0.4 @@ -4827,6 +4782,10 @@ snapshots: pkg-types: 1.3.1 ufo: 1.6.3 + mnemonist@0.39.8: + dependencies: + obliterator: 2.0.5 + motion-dom@12.38.0: dependencies: motion-utils: 12.36.0 @@ -4837,12 +4796,16 @@ snapshots: nanoid@3.3.11: {} - object-assign@4.1.1: {} + obliterator@2.0.5: {} package-manager-detector@1.6.0: {} pako@1.0.11: {} + pandemonium@2.4.1: + dependencies: + mnemonist: 0.39.8 + parse-entities@4.0.2: dependencies: '@types/unist': 2.0.11 @@ -4889,16 +4852,8 @@ snapshots: picocolors: 1.1.1 source-map-js: 1.2.1 - preact@10.29.1: {} - process-nextick-args@2.0.1: {} - prop-types@15.8.1: - dependencies: - loose-envify: 1.4.0 - object-assign: 4.1.1 - react-is: 16.13.1 - property-information@7.1.0: {} radix-ui@1.4.3(@types/react-dom@19.2.3(@types/react@19.2.14))(@types/react@19.2.14)(react-dom@19.2.4(react@19.2.4))(react@19.2.4): @@ -4969,13 +4924,6 @@ snapshots: react: 19.2.4 scheduler: 0.27.0 - react-force-graph-2d@1.29.1(react@19.2.4): - dependencies: - force-graph: 1.51.2 - prop-types: 15.8.1 - react: 19.2.4 - react-kapsule: 2.5.7(react@19.2.4) - react-hook-form@7.72.1(react@19.2.4): dependencies: react: 19.2.4 @@ -4991,15 +4939,8 @@ snapshots: react-dom: 19.2.4(react@19.2.4) typescript: 6.0.2 - react-is@16.13.1: {} - react-is@19.2.4: {} - react-kapsule@2.5.7(react@19.2.4): - dependencies: - jerrypick: 1.1.2 - react: 19.2.4 - react-markdown@10.1.0(@types/react@19.2.14)(react@19.2.4): dependencies: '@types/hast': 3.0.4 @@ -5208,6 +5149,13 @@ snapshots: setimmediate@1.0.5: {} + sigma@3.0.2(graphology-types@0.24.8): + dependencies: + events: 3.3.0 + graphology-utils: 2.5.2(graphology-types@0.24.8) + transitivePeerDependencies: + - graphology-types + source-map-js@1.2.1: {} space-separated-tokens@2.0.2: {} @@ -5239,8 +5187,6 @@ snapshots: tiny-invariant@1.3.3: {} - tinycolor2@1.6.0: {} - tinyexec@1.1.1: {} tinyglobby@0.2.15: diff --git a/ui/web/src/adapters/kg-graph.adapter.ts b/ui/web/src/adapters/kg-graph.adapter.ts index 947f3afa..d33bd5bf 100644 --- a/ui/web/src/adapters/kg-graph.adapter.ts +++ b/ui/web/src/adapters/kg-graph.adapter.ts @@ -1,6 +1,8 @@ +import Graph from "graphology"; import type { KGEntity, KGRelation } from "@/types/knowledge-graph"; +import { getNodeSize, truncateMiddle } from "@/components/graph/graph-utils"; -// Solid colors per entity type — used for both circle fill and glow +// Solid colors per entity type export const KG_TYPE_COLORS: Record = { person: "#E85D24", organization: "#ef4444", project: "#22c55e", product: "#f97316", technology: "#3b82f6", task: "#f59e0b", @@ -8,31 +10,6 @@ export const KG_TYPE_COLORS: Record = { }; export const KG_DEFAULT_COLOR = "#9ca3af"; -export interface GraphNode { - id: string; - name: string; - entityType: string; - color: string; - neighbors: Set; - linkIds: Set; - degree: number; - // Force-graph adds these at runtime - x?: number; - y?: number; -} - -export interface GraphLink { - id: string; - source: string; - target: string; - label: string; -} - -export interface GraphData { - nodes: GraphNode[]; - links: GraphLink[]; -} - /** Compute degree (edge count) for each entity id. */ export function computeDegreeMap(entities: KGEntity[], relations: KGRelation[]): Map { const deg = new Map(); @@ -44,47 +21,43 @@ export function computeDegreeMap(entities: KGEntity[], relations: KGRelation[]): return deg; } -/** - * Transform KGEntity[] + KGRelation[] into react-force-graph GraphData. - * Builds nodes with color/degree, links with neighbor/linkId sets populated. - */ -export function buildGraphData(entities: KGEntity[], allRelations: KGRelation[]): GraphData { +/** Build a Graphology graph from KG entities and relations. */ +export function buildKGGraph(entities: KGEntity[], allRelations: KGRelation[]): Graph { + const graph = new Graph({ multi: false, type: "directed" }); const entityIds = new Set(entities.map((e) => e.id)); const degreeMap = computeDegreeMap(entities, allRelations); - const nodes: GraphNode[] = entities.map((e) => ({ - id: e.id, - name: e.name, - entityType: e.entity_type, - color: KG_TYPE_COLORS[e.entity_type] ?? KG_DEFAULT_COLOR, - neighbors: new Set(), - linkIds: new Set(), - degree: degreeMap.get(e.id) ?? 0, - })); + // Add nodes (x/y assigned by container via circular layout before FA2) + for (const e of entities) { + if (!graph.hasNode(e.id)) { + const degree = degreeMap.get(e.id) ?? 0; + graph.addNode(e.id, { + label: truncateMiddle(e.name, 28), + x: 0, + y: 0, + size: getNodeSize(degree), + color: KG_TYPE_COLORS[e.entity_type] ?? KG_DEFAULT_COLOR, + entityType: e.entity_type, + }); + } + } - const nodeMap = new Map(nodes.map((n) => [n.id, n])); + // Add edges (straight arrows for KG) + for (const r of allRelations) { + if (entityIds.has(r.source_entity_id) && entityIds.has(r.target_entity_id)) { + if (!graph.hasEdge(r.source_entity_id, r.target_entity_id)) { + graph.addEdgeWithKey(r.id, r.source_entity_id, r.target_entity_id, { + label: r.relation_type.replace(/_/g, " "), + type: "curvedArrow", + }); + } + } + } - const links: GraphLink[] = allRelations - .filter((r) => entityIds.has(r.source_entity_id) && entityIds.has(r.target_entity_id)) - .map((r) => { - nodeMap.get(r.source_entity_id)?.neighbors.add(r.target_entity_id); - nodeMap.get(r.target_entity_id)?.neighbors.add(r.source_entity_id); - nodeMap.get(r.source_entity_id)?.linkIds.add(r.id); - nodeMap.get(r.target_entity_id)?.linkIds.add(r.id); - return { - id: r.id, - source: r.source_entity_id, - target: r.target_entity_id, - label: r.relation_type.replace(/_/g, " "), - }; - }); - - return { nodes, links }; + return graph; } -/** - * Limit entities to nodeLimit by degree centrality (highest-degree first). - */ +/** Limit entities to nodeLimit by degree centrality (highest-degree first). */ export function limitEntitiesByDegree( allEntities: KGEntity[], allRelations: KGRelation[], diff --git a/ui/web/src/adapters/vault-graph-adapter.ts b/ui/web/src/adapters/vault-graph-adapter.ts index 5acd027c..6c021e83 100644 --- a/ui/web/src/adapters/vault-graph-adapter.ts +++ b/ui/web/src/adapters/vault-graph-adapter.ts @@ -1,40 +1,19 @@ +import Graph from "graphology"; import type { VaultDocument, VaultLink } from "@/types/vault"; +import { getNodeSize, truncateMiddle } from "@/components/graph/graph-utils"; -// Colors per vault document type +// Colors per vault document type — dominant types (note 44%, media 54%) use +// softer tones; rare types use vivid accents for visibility. export const VAULT_TYPE_COLORS: Record = { - context: "#3b82f6", // blue - memory: "#8b5cf6", // purple - note: "#eab308", // yellow + context: "#6366f1", // indigo (rare, vivid) + memory: "#8b5cf6", // violet + note: "#14b8a6", // teal (dominant, soft) skill: "#22c55e", // green - episodic: "#f97316", // orange - media: "#ef4444", // red + episodic: "#f59e0b", // amber + media: "#ec4899", // pink (dominant, soft) }; const DEFAULT_COLOR = "#9ca3af"; -export interface VaultGraphNode { - id: string; - title: string; - docType: string; - color: string; - neighbors: Set; - linkIds: Set; - degree: number; - x?: number; - y?: number; -} - -export interface VaultGraphLink { - id: string; - source: string; - target: string; - label: string; -} - -export interface VaultGraphData { - nodes: VaultGraphNode[]; - links: VaultGraphLink[]; -} - /** Limit documents by degree centrality (highest-connected first). */ export function limitVaultDocsByDegree( docs: VaultDocument[], @@ -51,53 +30,49 @@ export function limitVaultDocsByDegree( return [...docs].sort((a, b) => (deg.get(b.id) ?? 0) - (deg.get(a.id) ?? 0)).slice(0, nodeLimit); } -/** Build graph data from vault documents and their links. */ -export function buildVaultGraphData( +/** Build a Graphology graph from vault documents and their links. */ +export function buildVaultGraph( documents: VaultDocument[], links: VaultLink[], -): VaultGraphData { +): Graph { + const graph = new Graph({ multi: false, type: "directed" }); const docIds = new Set(documents.map((d) => d.id)); - // Build degree map + // Pre-compute degree map const degreeMap = new Map(); for (const link of links) { - if (docIds.has(link.from_doc_id)) { + if (docIds.has(link.from_doc_id)) degreeMap.set(link.from_doc_id, (degreeMap.get(link.from_doc_id) ?? 0) + 1); - } - if (docIds.has(link.to_doc_id)) { + if (docIds.has(link.to_doc_id)) degreeMap.set(link.to_doc_id, (degreeMap.get(link.to_doc_id) ?? 0) + 1); - } } - const nodes: VaultGraphNode[] = documents.map((d) => ({ - id: d.id, - title: d.title || d.path.split("/").pop() || d.id.slice(0, 8), - docType: d.doc_type, - color: VAULT_TYPE_COLORS[d.doc_type] ?? DEFAULT_COLOR, - neighbors: new Set(), - linkIds: new Set(), - degree: degreeMap.get(d.id) ?? 0, - })); - - const nodeMap = new Map(nodes.map((n) => [n.id, n])); - - // Only include links where both endpoints exist in our document set - const graphLinks: VaultGraphLink[] = []; - for (const link of links) { - const src = nodeMap.get(link.from_doc_id); - const tgt = nodeMap.get(link.to_doc_id); - if (!src || !tgt) continue; - src.neighbors.add(tgt.id); - tgt.neighbors.add(src.id); - src.linkIds.add(link.id); - tgt.linkIds.add(link.id); - graphLinks.push({ - id: link.id, - source: link.from_doc_id, - target: link.to_doc_id, - label: link.link_type, + // Add nodes (x/y assigned by container via circular layout before FA2) + for (const doc of documents) { + const degree = degreeMap.get(doc.id) ?? 0; + const rawLabel = doc.title || doc.path.split("/").pop() || doc.id.slice(0, 8); + graph.addNode(doc.id, { + label: truncateMiddle(rawLabel, 28), + x: 0, + y: 0, + size: getNodeSize(degree), + color: VAULT_TYPE_COLORS[doc.doc_type] ?? DEFAULT_COLOR, + docType: doc.doc_type, }); } - return { nodes, links: graphLinks }; + // Add edges (only where both endpoints exist) + for (const link of links) { + if (docIds.has(link.from_doc_id) && docIds.has(link.to_doc_id)) { + // Avoid duplicate edges for same source→target + if (!graph.hasEdge(link.from_doc_id, link.to_doc_id)) { + graph.addEdgeWithKey(link.id, link.from_doc_id, link.to_doc_id, { + label: link.link_type, + type: "curvedArrow", + }); + } + } + } + + return graph; } diff --git a/ui/web/src/api/http-client.ts b/ui/web/src/api/http-client.ts index 2418ec39..0e9a3866 100644 --- a/ui/web/src/api/http-client.ts +++ b/ui/web/src/api/http-client.ts @@ -84,10 +84,10 @@ export class HttpClient { if (!res.ok) { const err = await res.json().catch(() => ({ error: res.statusText })); - throw new ApiError( - err.code ?? "HTTP_ERROR", - err.error ?? err.message ?? res.statusText, - ); + const nested = typeof err.error === "object" && err.error !== null ? err.error : null; + const code = nested?.code ?? err.code ?? "HTTP_ERROR"; + const message = nested?.message ?? (typeof err.error === "string" ? err.error : null) ?? err.message ?? res.statusText; + throw new ApiError(code, message); } return res.json() as Promise; @@ -140,13 +140,14 @@ export class HttpClient { if (!res.ok) { const err = await res.json().catch(() => ({ error: res.statusText })); - if (res.status === 401 || err.code === "TENANT_ACCESS_REVOKED") { + // Backend wraps errors as { "error": { "code": "...", "message": "..." } } + const nested = typeof err.error === "object" && err.error !== null ? err.error : null; + const code = nested?.code ?? err.code ?? "HTTP_ERROR"; + const message = nested?.message ?? (typeof err.error === "string" ? err.error : null) ?? err.message ?? res.statusText; + if (res.status === 401 || code === "TENANT_ACCESS_REVOKED") { this.onAuthFailure?.(); } - throw new ApiError( - err.code ?? "HTTP_ERROR", - err.error ?? err.message ?? res.statusText, - ); + throw new ApiError(code, message); } return res.json() as Promise; diff --git a/ui/web/src/components/chat/chat-top-bar.tsx b/ui/web/src/components/chat/chat-top-bar.tsx index 527977a5..ddac7a63 100644 --- a/ui/web/src/components/chat/chat-top-bar.tsx +++ b/ui/web/src/components/chat/chat-top-bar.tsx @@ -1,9 +1,11 @@ import { useState, useEffect } from "react"; +import { useTranslation } from "react-i18next"; import { Loader2, Bot, Users, PanelRightOpen, PanelRightClose } from "lucide-react"; import { useHttp } from "@/hooks/use-ws"; import { useAuthStore } from "@/stores/use-auth-store"; import type { RunActivity, ActiveTeamTask } from "@/types/chat"; import type { AgentData } from "@/types/agent"; +import type { SessionInfo } from "@/types/session"; interface ChatTopBarProps { agentId: string; @@ -13,6 +15,8 @@ interface ChatTopBarProps { teamTasks: ActiveTeamTask[]; onToggleTaskPanel?: () => void; taskPanelOpen?: boolean; + /** Current session — when provided, the bar renders a context-usage badge. */ + session?: SessionInfo | null; } const phaseLabels: Record = { @@ -24,8 +28,9 @@ const phaseLabels: Record = { leader_processing: "Processing team results…", }; -export function ChatTopBar({ agentId, isRunning, isBusy, activity, teamTasks, onToggleTaskPanel, taskPanelOpen }: ChatTopBarProps) { +export function ChatTopBar({ agentId, isRunning, isBusy, activity, teamTasks, onToggleTaskPanel, taskPanelOpen, session }: ChatTopBarProps) { const http = useHttp(); + const { t } = useTranslation("chat"); const connected = useAuthStore((s) => s.connected); const [agent, setAgent] = useState<{ name: string; emoji?: string } | null>(null); @@ -51,6 +56,31 @@ export function ChatTopBar({ agentId, isRunning, isBusy, activity, teamTasks, on const emoji = agent?.emoji; const PanelIcon = taskPanelOpen ? PanelRightClose : PanelRightOpen; + // Context-usage badge: only renders when the caller passes a session with + // both estimatedTokens (Phase 4 ContextStage output) and contextWindow. + // `percent` drives the color ramp so operators spot near-limit sessions. + const usage = (() => { + if (!session || !session.contextWindow || session.contextWindow <= 0) { + return null; + } + const used = session.estimatedTokens ?? 0; + const max = session.contextWindow; + const percent = Math.min(100, Math.round((used / max) * 100)); + const color = + percent >= 90 ? "text-destructive" : percent >= 75 ? "text-amber-600 dark:text-amber-400" : "text-muted-foreground"; + return { used, max, percent, color }; + })(); + + // Last compaction timestamp ships in sessions.metadata JSONB (Phase 5 follow-up, + // keyed "last_compaction_at"). Parsed lazily so bad data doesn't crash the bar. + const lastCompaction = (() => { + const raw = session?.metadata?.last_compaction_at; + if (!raw) return null; + const d = new Date(raw); + if (isNaN(d.getTime())) return null; + return d; + })(); + return ( @@ -63,6 +93,23 @@ export function ChatTopBar({ agentId, isRunning, isBusy, activity, teamTasks, on + {usage && ( + + + {usage.used.toLocaleString()}/{usage.max.toLocaleString()} + + ({usage.percent}%) + + )} {isRunning ? ( {activity ? phaseLabels[activity.phase] : "Running…"} diff --git a/ui/web/src/components/graph/graph-utils.ts b/ui/web/src/components/graph/graph-utils.ts new file mode 100644 index 00000000..a562580c --- /dev/null +++ b/ui/web/src/components/graph/graph-utils.ts @@ -0,0 +1,32 @@ +/** Degree-based node sizing: 4-12px range. Logarithmic scale for dense hubs. */ +export function getNodeSize(degree: number): number { + if (degree === 0) return 4; + return 4 + Math.min(Math.log2(degree + 1) * 2, 8); +} + +/** + * Truncate a long string by removing the middle and inserting an ellipsis. + * Example: truncateMiddle("Model Steering System Benchmark", 20) → "Model Ster…hmark" + */ +export function truncateMiddle(str: string, maxLength = 28): string { + if (!str || str.length <= maxLength) return str; + const keepStart = Math.ceil((maxLength - 1) * 0.6); + const keepEnd = Math.floor((maxLength - 1) * 0.4); + return `${str.slice(0, keepStart)}…${str.slice(-keepEnd)}`; +} + +/** Sigma settings constants for consistent look across both graph views */ +export const SIGMA_SETTINGS = { + /** Labels hidden for nodes smaller than this on screen */ + labelRenderedSizeThreshold: 8, + /** Lower = fewer labels shown (avoids overlap) */ + labelDensity: 0.12, + /** Grid cell size for label collision avoidance */ + labelGridCellSize: 120, + /** Default edge color (overridden per-theme) */ + defaultEdgeColor: "#334155", + /** Minimum camera ratio (most zoomed in) */ + minCameraRatio: 0.02, + /** Maximum camera ratio (most zoomed out) */ + maxCameraRatio: 8, +} as const; diff --git a/ui/web/src/components/graph/sigma-graph-container.tsx b/ui/web/src/components/graph/sigma-graph-container.tsx new file mode 100644 index 00000000..1731a217 --- /dev/null +++ b/ui/web/src/components/graph/sigma-graph-container.tsx @@ -0,0 +1,371 @@ +import { useRef, useEffect, useCallback, useState } from "react"; +import Sigma from "sigma"; +import { EdgeCurvedArrowProgram } from "@sigma/edge-curve"; +import { EdgeArrowProgram } from "sigma/rendering"; +import forceAtlas2 from "graphology-layout-forceatlas2"; +import noverlap from "graphology-layout-noverlap"; +import type Graph from "graphology"; +import { useUiStore } from "@/stores/use-ui-store"; +import { SIGMA_SETTINGS } from "./graph-utils"; + +export type EdgeType = "curvedArrow" | "arrow"; + +export interface SigmaGraphContainerProps { + graph: Graph; + edgeType?: EdgeType; + selectedNodeId?: string | null; + onNodeSelect?: (nodeId: string | null) => void; + onNodeDoubleClick?: (nodeId: string) => void; + /** Called when Sigma instance is ready (or destroyed) */ + onSigmaReady?: (sigma: Sigma | null) => void; + /** Compact mode for embedded mini-graphs (no layout, smaller labels) */ + compact?: boolean; + /** Node types to hide (from filter component) */ + hiddenTypes?: Set; +} + +/** Theme-aware colors */ +function useThemeColors() { + const theme = useUiStore((s) => s.theme); + const isDark = + theme === "dark" || + (theme === "system" && + window.matchMedia("(prefers-color-scheme: dark)").matches); + return { + isDark, + labelColor: isDark ? "#e2e8f0" : "#1e293b", + // Base edge color — visible but not dominant + edgeColor: isDark ? "#47556966" : "#94a3b8cc", + // Dim color for non-active edges + dimEdgeColor: isDark ? "#47556933" : "#cbd5e166", + // Highlight for active neighborhood edges + highlightEdgeColor: isDark ? "#a1a1aa" : "#64748b", + }; +} + +export function SigmaGraphContainer({ + graph, + edgeType = "arrow", + selectedNodeId, + onNodeSelect, + onNodeDoubleClick, + onSigmaReady, + compact = false, + hiddenTypes, +}: SigmaGraphContainerProps) { + const containerRef = useRef(null); + const internalSigmaRef = useRef(null); + // Incremented when sigma instance changes — used to trigger event handler registration. + const [sigmaVersion, setSigmaVersion] = useState(0); + const [hoveredNode, setHoveredNode] = useState(null); + // Pulse phase for animated highlighted edges (0..1, cycles) + const [pulsePhase, setPulsePhase] = useState(0); + const { labelColor, edgeColor, dimEdgeColor, highlightEdgeColor } = useThemeColors(); + + const setSigmaRef = useCallback( + (instance: Sigma | null) => { + internalSigmaRef.current = instance; + setSigmaVersion((v) => v + 1); + onSigmaReady?.(instance); + }, + [onSigmaReady], + ); + + // --- Initialize Sigma with PRE-COMPUTED FA2 layout (no shake, no worker) --- + useEffect(() => { + if (!containerRef.current || graph.order === 0) return; + + // Unified adaptive FA2 layout — no density branching. + // Params scale with orphan ratio (disconnected nodes / total nodes). + if (graph.order > 1) { + // Random disc init — uniform area distribution for all graph types. + const spread = Math.sqrt(graph.order) * 20; + const nodes = graph.nodes(); + for (let i = 0; i < nodes.length; i++) { + const angle = Math.random() * Math.PI * 2; + const r = Math.sqrt(Math.random()) * spread; + graph.setNodeAttribute(nodes[i], "x", Math.cos(angle) * r); + graph.setNodeAttribute(nodes[i], "y", Math.sin(angle) * r); + } + + // Orphan ratio drives adaptive parameters: + // High orphan → stronger gravity (pull orphans to center), tighter layout + // Low orphan → link forces dominate, more spread for cluster readability + let orphanCount = 0; + for (const node of nodes) { + if (graph.degree(node) === 0) orphanCount++; + } + const orphanRatio = orphanCount / graph.order; + const gravity = 0.5 + orphanRatio * 2.0; + const scalingRatio = 4.0 - orphanRatio * 2.0; + const iterations = graph.order < 100 ? 500 : graph.order < 500 ? 300 : 200; + + forceAtlas2.assign(graph, { + iterations, + settings: { + linLogMode: true, + outboundAttractionDistribution: true, + gravity, + scalingRatio, + adjustSizes: true, + strongGravityMode: true, + slowDown: 8, + barnesHutOptimize: graph.order > 100, + edgeWeightInfluence: 0, + }, + }); + + // Noverlap only for connected-heavy graphs (prevents overlap in dense clusters). + // Skip for high-orphan graphs to avoid grid artifacts. + if (orphanRatio < 0.3) { + noverlap.assign(graph, { + maxIterations: 30, + settings: { margin: 2, ratio: 1.02, speed: 3, gridSize: 20 }, + }); + } + } + + const edgePrograms: Record = { + arrow: EdgeArrowProgram, + curvedArrow: EdgeCurvedArrowProgram as unknown as typeof EdgeArrowProgram, + }; + + const sigma = new Sigma(graph, containerRef.current, { + allowInvalidContainer: true, + renderLabels: true, + labelRenderedSizeThreshold: compact ? 14 : SIGMA_SETTINGS.labelRenderedSizeThreshold, + labelDensity: compact ? 0.05 : SIGMA_SETTINGS.labelDensity, + labelGridCellSize: SIGMA_SETTINGS.labelGridCellSize, + labelColor: { color: labelColor }, + defaultEdgeColor: edgeColor, + defaultEdgeType: edgeType, + edgeProgramClasses: edgePrograms, + minCameraRatio: SIGMA_SETTINGS.minCameraRatio, + maxCameraRatio: SIGMA_SETTINGS.maxCameraRatio, + labelFont: "Inter, system-ui, sans-serif", + labelSize: compact ? 10 : 12, + zoomingRatio: 1.3, + // Enable z-index sorting — required for edge/node `zIndex` attr to affect render order + zIndex: true, + }); + + setSigmaRef(sigma); + + // Fit viewport to graph on next frame (after initial render) + requestAnimationFrame(() => { + sigma.getCamera().animatedReset({ duration: 300 }); + }); + + return () => { + sigma.kill(); + // Only clear external ref if it still points to THIS sigma (concurrent mode safety) + if (internalSigmaRef.current === sigma) { + internalSigmaRef.current = null; + onSigmaReady?.(null); + } + }; + }, [graph, edgeType, compact]); // eslint-disable-line react-hooks/exhaustive-deps + + // --- Update theme colors without re-init --- + useEffect(() => { + const sigma = internalSigmaRef.current; + if (!sigma) return; + sigma.setSetting("labelColor", { color: labelColor }); + sigma.setSetting("defaultEdgeColor", edgeColor); + sigma.refresh(); + }, [labelColor, edgeColor]); + + // Compute multi-hop neighborhood (BFS, 2 hops) for active node + const neighborhoodRef = useRef<{ nodes: Set; edges: Set } | null>(null); + useEffect(() => { + const active = selectedNodeId || hoveredNode; + if (!active || !graph.hasNode(active)) { + neighborhoodRef.current = null; + return; + } + const nodes = new Set([active]); + const edges = new Set(); + const MAX_HOPS = 2; + let frontier: string[] = [active]; + for (let hop = 0; hop < MAX_HOPS; hop++) { + const next: string[] = []; + for (const n of frontier) { + graph.forEachEdge(n, (edge, _attrs, source, target) => { + edges.add(edge); + const other = source === n ? target : source; + if (!nodes.has(other)) { + nodes.add(other); + next.push(other); + } + }); + } + frontier = next; + if (frontier.length === 0) break; + } + neighborhoodRef.current = { nodes, edges }; + }, [selectedNodeId, hoveredNode, graph]); + + // --- Unified node/edge reducers: filter + subtle hover highlight (no dimming) --- + useEffect(() => { + const sigma = internalSigmaRef.current; + if (!sigma) return; + + const getNodeType = (attrs: Record) => + (attrs.docType || attrs.entityType || "other") as string; + + sigma.setSetting("nodeReducer", (node, data) => { + // Filter: hide nodes of hidden types + if (hiddenTypes?.size && hiddenTypes.has(getNodeType(data))) { + return { ...data, hidden: true }; + } + + const activeNode = selectedNodeId || hoveredNode; + if (!activeNode) return data; + + const hood = neighborhoodRef.current; + if (!hood) return data; + + if (node === activeNode) { + // Active node: bring to top, show label + return { ...data, zIndex: 3, forceLabel: true }; + } + if (hood.nodes.has(node)) { + // Within 2-hop neighborhood: show label, slightly raised z-index + return { ...data, zIndex: 2, forceLabel: true }; + } + // Outside neighborhood: keep color/size, no dimming — just lower z-index + return { ...data, zIndex: 0 }; + }); + + sigma.setSetting("edgeReducer", (edge, data) => { + // Filter: hide edges connected to hidden node types + if (hiddenTypes?.size) { + const srcAttrs = graph.getNodeAttributes(graph.source(edge)); + const tgtAttrs = graph.getNodeAttributes(graph.target(edge)); + if (hiddenTypes.has(getNodeType(srcAttrs)) || hiddenTypes.has(getNodeType(tgtAttrs))) { + return { ...data, hidden: true }; + } + } + + const activeNode = selectedNodeId || hoveredNode; + if (!activeNode) return data; + + const hood = neighborhoodRef.current; + if (!hood) return data; + + if (hood.edges.has(edge)) { + // Edges within neighborhood: thin + subtle OPACITY pulse (not size). + const alpha = Math.round(200 + Math.sin(pulsePhase * Math.PI * 2) * 40); + const alphaHex = Math.max(0, Math.min(255, alpha)).toString(16).padStart(2, "0"); + const pulsedColor = `${highlightEdgeColor}${alphaHex}`; + return { ...data, color: pulsedColor, size: 1, zIndex: 2 }; + } + // Non-related edges: HIDE entirely so active edges are clearly visible + return { ...data, hidden: true }; + }); + + sigma.refresh(); + }, [selectedNodeId, hoveredNode, graph, highlightEdgeColor, dimEdgeColor, hiddenTypes]); + + // --- Pulse animation for highlighted edges (only runs when a node is active) --- + // Respects prefers-reduced-motion — skips animation entirely for accessibility + useEffect(() => { + const active = selectedNodeId || hoveredNode; + if (!active) return; + + // Honor user's reduced-motion preference + const mediaQuery = window.matchMedia("(prefers-reduced-motion: reduce)"); + if (mediaQuery.matches) return; + + let rafId = 0; + const start = performance.now(); + const PULSE_PERIOD_MS = 1800; // slower, gentler pulse + const tick = () => { + const elapsed = performance.now() - start; + setPulsePhase((elapsed % PULSE_PERIOD_MS) / PULSE_PERIOD_MS); + rafId = requestAnimationFrame(tick); + }; + rafId = requestAnimationFrame(tick); + return () => cancelAnimationFrame(rafId); + }, [selectedNodeId, hoveredNode]); + + // Add pulsePhase to reducer deps so edges re-render on pulse + useEffect(() => { + const sigma = internalSigmaRef.current; + if (!sigma) return; + sigma.refresh({ skipIndexation: true }); + }, [pulsePhase]); + + // --- Event handlers: use refs for values that change frequently to avoid + // re-registering Sigma listeners (which drops in-flight double-click events) --- + const selectedNodeIdRef = useRef(selectedNodeId); + selectedNodeIdRef.current = selectedNodeId; + const onNodeSelectRef = useRef(onNodeSelect); + onNodeSelectRef.current = onNodeSelect; + const onNodeDoubleClickRef = useRef(onNodeDoubleClick); + onNodeDoubleClickRef.current = onNodeDoubleClick; + + useEffect(() => { + const sigma = internalSigmaRef.current; + if (!sigma) return; + + const handleEnterNode = ({ node }: { node: string }) => { + setHoveredNode(node); + if (containerRef.current) containerRef.current.style.cursor = "pointer"; + }; + + const handleLeaveNode = () => { + setHoveredNode(null); + if (containerRef.current) containerRef.current.style.cursor = "default"; + }; + + const handleClickNode = ({ node }: { node: string }) => { + onNodeSelectRef.current?.(node === selectedNodeIdRef.current ? null : node); + }; + + const handleDoubleClickNode = ({ node, event }: { node: string; event: { preventSigmaDefault?: () => void } }) => { + // Prevent Sigma's default zoom-in behavior on double-click + event.preventSigmaDefault?.(); + onNodeDoubleClickRef.current?.(node); + }; + + const handleClickStage = () => { + onNodeSelectRef.current?.(null); + }; + + sigma.on("enterNode", handleEnterNode); + sigma.on("leaveNode", handleLeaveNode); + sigma.on("clickNode", handleClickNode); + sigma.on("doubleClickNode", handleDoubleClickNode); + sigma.on("clickStage", handleClickStage); + + return () => { + sigma.off("enterNode", handleEnterNode); + sigma.off("leaveNode", handleLeaveNode); + sigma.off("clickNode", handleClickNode); + sigma.off("doubleClickNode", handleDoubleClickNode); + sigma.off("clickStage", handleClickStage); + }; + }, [sigmaVersion]); // re-register only when sigma instance changes + + // NOTE: Click on node NO LONGER moves camera. + // Camera only animates for explicit user actions (search, fit-to-view, keyboard F). + // This matches the old force-graph behavior where clicking just highlights. + + // No-data state + if (graph.order === 0) { + return ( + + No data to display + + ); + } + + return ( + + ); +} diff --git a/ui/web/src/components/graph/sigma-graph-controls.tsx b/ui/web/src/components/graph/sigma-graph-controls.tsx new file mode 100644 index 00000000..2276e51a --- /dev/null +++ b/ui/web/src/components/graph/sigma-graph-controls.tsx @@ -0,0 +1,87 @@ +import { useCallback, useEffect, useState } from "react"; +import type Sigma from "sigma"; +import { ZoomIn, ZoomOut, Maximize2 } from "lucide-react"; +import { Button } from "@/components/ui/button"; + +const NODE_LIMIT_OPTIONS = [100, 200, 500, 1000, 5000] as const; + +interface SigmaGraphControlsProps { + sigma: Sigma | null; + nodeLimit: number; + isLimited: boolean; + onNodeLimitChange: (limit: number) => void; + /** i18n labels — caller provides pre-formatted translated strings */ + labels: { + nodes: string; + edges: string; + limitNote?: string; + }; +} + +export function SigmaGraphControls({ + sigma, + nodeLimit, + isLimited, + onNodeLimitChange, + labels, +}: SigmaGraphControlsProps) { + const [zoomPercent, setZoomPercent] = useState(100); + + // Track zoom level from camera events + useEffect(() => { + if (!sigma) return; + const camera = sigma.getCamera(); + const update = () => { + // ratio < 1 = zoomed in, ratio > 1 = zoomed out. Invert for display. + setZoomPercent(Math.round((1 / camera.ratio) * 100)); + }; + update(); + camera.on("updated", update); + return () => { camera.off("updated", update); }; + }, [sigma]); + + const handleZoomIn = useCallback(() => { + sigma?.getCamera().animatedZoom({ duration: 200 }); + }, [sigma]); + + const handleZoomOut = useCallback(() => { + sigma?.getCamera().animatedUnzoom({ duration: 200 }); + }, [sigma]); + + const handleFitToView = useCallback(() => { + sigma?.getCamera().animatedReset({ duration: 300 }); + }, [sigma]); + + return ( + + {labels.nodes} + {labels.edges} + {isLimited && labels.limitNote && ( + · {labels.limitNote} + )} + + + + + + {zoomPercent}% + + + + + onNodeLimitChange(Number(e.target.value))} + className="h-5 rounded border bg-background px-1 text-base md:text-2xs" + aria-label="Node limit" + > + {NODE_LIMIT_OPTIONS.map((n) => ( + {n} nodes + ))} + + + + + + ); +} diff --git a/ui/web/src/components/graph/sigma-graph-filters.tsx b/ui/web/src/components/graph/sigma-graph-filters.tsx new file mode 100644 index 00000000..1f949c3b --- /dev/null +++ b/ui/web/src/components/graph/sigma-graph-filters.tsx @@ -0,0 +1,122 @@ +import { useCallback, useMemo, useEffect, useRef } from "react"; +import type Graph from "graphology"; +import { Filter, ChevronDown, ChevronUp } from "lucide-react"; +import { Button } from "@/components/ui/button"; + +interface TypeCount { + type: string; + color: string; + count: number; +} + +interface SigmaGraphFiltersProps { + graph: Graph; + /** Map of type → color for the legend dots */ + typeColors: Record; + defaultColor?: string; + /** Currently hidden types (controlled state) */ + hiddenTypes: Set; + onHiddenTypesChange: (types: Set) => void; + collapsed: boolean; + onCollapsedChange: (collapsed: boolean) => void; +} + +export function SigmaGraphFilters({ + graph, typeColors, defaultColor = "#9ca3af", + hiddenTypes, onHiddenTypesChange, + collapsed, onCollapsedChange, +}: SigmaGraphFiltersProps) { + const containerRef = useRef(null); + + // Memoize type counts to avoid iterating all nodes on every render + const typeCounts = useMemo(() => { + if (graph.order === 0) return []; + const counts = new Map(); + graph.forEachNode((_node, attrs) => { + const t = (attrs.docType || attrs.entityType || "other") as string; + counts.set(t, (counts.get(t) ?? 0) + 1); + }); + const result: TypeCount[] = []; + for (const [type, count] of counts) { + result.push({ type, color: typeColors[type] ?? defaultColor, count }); + } + return result.sort((a, b) => b.count - a.count); + }, [graph, typeColors, defaultColor]); + + const toggleType = useCallback((type: string) => { + const next = new Set(hiddenTypes); + if (next.has(type)) next.delete(type); + else next.add(type); + onHiddenTypesChange(next); + }, [hiddenTypes, onHiddenTypesChange]); + + // Close dropdown on outside click OR Escape key + useEffect(() => { + if (collapsed) return; + const handleClick = (e: MouseEvent) => { + if (containerRef.current && !containerRef.current.contains(e.target as Node)) { + onCollapsedChange(true); + } + }; + const handleKeyDown = (e: KeyboardEvent) => { + if (e.key === "Escape") { + onCollapsedChange(true); + } + }; + document.addEventListener("mousedown", handleClick); + document.addEventListener("keydown", handleKeyDown); + return () => { + document.removeEventListener("mousedown", handleClick); + document.removeEventListener("keydown", handleKeyDown); + }; + }, [collapsed, onCollapsedChange]); + + if (typeCounts.length === 0) return null; + + return ( + + onCollapsedChange(!collapsed)} + aria-label={`Filter by type${hiddenTypes.size > 0 ? ` (${hiddenTypes.size} hidden)` : ""}`} + aria-expanded={!collapsed} + aria-haspopup="menu" + > + + Filter + {hiddenTypes.size > 0 && ( + + {hiddenTypes.size} hidden + + )} + {collapsed ? : } + + + {!collapsed && ( + + {typeCounts.map(({ type, color, count }) => { + const hidden = hiddenTypes.has(type); + return ( + toggleType(type)} + className={`flex items-center gap-2 px-2 py-1 rounded text-xs hover:bg-accent transition-opacity ${ + hidden ? "opacity-40" : "" + }`} + > + + {type} + {count} + + ); + })} + + )} + + ); +} diff --git a/ui/web/src/components/graph/sigma-graph-keyboard-help.tsx b/ui/web/src/components/graph/sigma-graph-keyboard-help.tsx new file mode 100644 index 00000000..c86a693a --- /dev/null +++ b/ui/web/src/components/graph/sigma-graph-keyboard-help.tsx @@ -0,0 +1,106 @@ +import { useState, useEffect, useRef } from "react"; +import { Keyboard, X } from "lucide-react"; +import { Button } from "@/components/ui/button"; + +interface Shortcut { + keys: string[]; + description: string; +} + +const SHORTCUTS: Shortcut[] = [ + { keys: ["+"], description: "Zoom in" }, + { keys: ["-"], description: "Zoom out" }, + { keys: ["R"], description: "Reset view (fit all)" }, + { keys: ["F"], description: "Focus selected node" }, + { keys: ["Esc"], description: "Deselect" }, + { keys: ["/"], description: "Focus search" }, + { keys: ["?"], description: "Show this help" }, +]; + +/** + * Keyboard shortcuts help popover. + * Triggered by `?` key or by clicking the keyboard icon. + */ +export function SigmaGraphKeyboardHelp() { + const [open, setOpen] = useState(false); + const containerRef = useRef(null); + + // Global `?` key handler to open help (when not typing in input) + useEffect(() => { + const handler = (e: KeyboardEvent) => { + const tag = (e.target as HTMLElement)?.tagName; + if (tag === "INPUT" || tag === "TEXTAREA" || tag === "SELECT") return; + if (e.key === "?") { + e.preventDefault(); + setOpen((prev) => !prev); + } else if (e.key === "Escape" && open) { + setOpen(false); + } + }; + document.addEventListener("keydown", handler); + return () => document.removeEventListener("keydown", handler); + }, [open]); + + // Outside click to dismiss + useEffect(() => { + if (!open) return; + const handler = (e: MouseEvent) => { + if (containerRef.current && !containerRef.current.contains(e.target as Node)) { + setOpen(false); + } + }; + document.addEventListener("mousedown", handler); + return () => document.removeEventListener("mousedown", handler); + }, [open]); + + return ( + + setOpen((p) => !p)} + aria-label="Keyboard shortcuts" + title="Keyboard shortcuts (?)" + > + + + + {open && ( + + + Keyboard shortcuts + setOpen(false)} + className="p-0.5 rounded hover:bg-accent" + aria-label="Close shortcuts help" + > + + + + + {SHORTCUTS.map(({ keys, description }) => ( + + {description} + + {keys.map((k) => ( + + {k} + + ))} + + + ))} + + + )} + + ); +} diff --git a/ui/web/src/components/graph/sigma-graph-minimap.tsx b/ui/web/src/components/graph/sigma-graph-minimap.tsx new file mode 100644 index 00000000..b53a2555 --- /dev/null +++ b/ui/web/src/components/graph/sigma-graph-minimap.tsx @@ -0,0 +1,177 @@ +import { useRef, useEffect, useCallback } from "react"; +import type Sigma from "sigma"; +import type Graph from "graphology"; + +interface SigmaGraphMinimapProps { + sigma: Sigma | null; + graph: Graph; + /** Minimap canvas size in pixels */ + size?: number; +} + +/** + * Lightweight canvas-overlay minimap. + * Draws all nodes as small dots + a viewport rectangle showing the current camera view. + * Click to jump, no drag (KISS). + */ +export function SigmaGraphMinimap({ sigma, graph, size = 140 }: SigmaGraphMinimapProps) { + const canvasRef = useRef(null); + const rafRef = useRef(0); + + // Compute graph bounding box from node positions + const getBounds = useCallback(() => { + let minX = Infinity, minY = Infinity, maxX = -Infinity, maxY = -Infinity; + graph.forEachNode((_node, attrs) => { + const x = attrs.x as number; + const y = attrs.y as number; + if (x < minX) minX = x; + if (x > maxX) maxX = x; + if (y < minY) minY = y; + if (y > maxY) maxY = y; + }); + // Add padding + const padX = (maxX - minX) * 0.1 || 50; + const padY = (maxY - minY) * 0.1 || 50; + return { minX: minX - padX, minY: minY - padY, maxX: maxX + padX, maxY: maxY + padY }; + }, [graph]); + + // Draw minimap + const draw = useCallback(() => { + const canvas = canvasRef.current; + const ctx = canvas?.getContext("2d"); + if (!canvas || !ctx || !sigma || graph.order === 0) return; + + const dpr = window.devicePixelRatio || 1; + canvas.width = size * dpr; + canvas.height = size * dpr; + ctx.scale(dpr, dpr); + + const bounds = getBounds(); + const bw = bounds.maxX - bounds.minX; + const bh = bounds.maxY - bounds.minY; + const scale = Math.min(size / bw, size / bh); + const offsetX = (size - bw * scale) / 2; + const offsetY = (size - bh * scale) / 2; + + // Clear + ctx.clearRect(0, 0, size, size); + + // Background + ctx.fillStyle = "rgba(0,0,0,0.03)"; + ctx.fillRect(0, 0, size, size); + + // Draw nodes as 1.5px dots + graph.forEachNode((_node, attrs) => { + const x = ((attrs.x as number) - bounds.minX) * scale + offsetX; + const y = ((attrs.y as number) - bounds.minY) * scale + offsetY; + ctx.beginPath(); + ctx.arc(x, y, 1.5, 0, Math.PI * 2); + ctx.fillStyle = (attrs.color as string) || "#9ca3af"; + ctx.fill(); + }); + + // Draw viewport rectangle — use viewportToGraph to derive the 4 corners + // in graph space, then map to minimap space + const { width, height } = sigma.getDimensions(); + const topLeft = sigma.viewportToGraph({ x: 0, y: 0 }); + const bottomRight = sigma.viewportToGraph({ x: width, y: height }); + + const vx1 = (topLeft.x - bounds.minX) * scale + offsetX; + const vy1 = (topLeft.y - bounds.minY) * scale + offsetY; + const vx2 = (bottomRight.x - bounds.minX) * scale + offsetX; + const vy2 = (bottomRight.y - bounds.minY) * scale + offsetY; + const vw = vx2 - vx1; + const vh = vy2 - vy1; + + ctx.strokeStyle = "rgba(59, 130, 246, 0.7)"; + ctx.lineWidth = 1.5; + ctx.strokeRect(vx1, vy1, vw, vh); + ctx.fillStyle = "rgba(59, 130, 246, 0.08)"; + ctx.fillRect(vx1, vy1, vw, vh); + }, [sigma, graph, size, getBounds]); + + // Redraw on camera changes + useEffect(() => { + if (!sigma) return; + const camera = sigma.getCamera(); + + // Throttled redraw — skip if already scheduled + let scheduled = false; + const scheduleRedraw = () => { + if (scheduled) return; + scheduled = true; + rafRef.current = requestAnimationFrame(() => { + scheduled = false; + draw(); + }); + }; + + // Initial draw + scheduleRedraw(); + + // Redraw on camera updates (pan/zoom) AND after every sigma render + // (so FA2 layout position updates are reflected) + camera.on("updated", scheduleRedraw); + sigma.on("afterRender", scheduleRedraw); + + return () => { + camera.off("updated", scheduleRedraw); + sigma.off("afterRender", scheduleRedraw); + cancelAnimationFrame(rafRef.current); + }; + }, [sigma, draw]); + + // Click to jump: find nearest node to click position and center camera on it + const handleClick = useCallback( + (e: React.MouseEvent) => { + if (!sigma || graph.order === 0) return; + const rect = canvasRef.current?.getBoundingClientRect(); + if (!rect) return; + + const bounds = getBounds(); + const bw = bounds.maxX - bounds.minX; + const bh = bounds.maxY - bounds.minY; + const scale = Math.min(size / bw, size / bh); + const offsetX = (size - bw * scale) / 2; + const offsetY = (size - bh * scale) / 2; + + // Convert click position to graph coordinates + const clickX = e.clientX - rect.left; + const clickY = e.clientY - rect.top; + const targetGraphX = (clickX - offsetX) / scale + bounds.minX; + const targetGraphY = (clickY - offsetY) / scale + bounds.minY; + + // Find nearest node (avoids coordinate-system issues with camera.animate) + let nearestId: string | null = null; + let minDist = Infinity; + graph.forEachNode((id, attrs) => { + const dx = (attrs.x as number) - targetGraphX; + const dy = (attrs.y as number) - targetGraphY; + const d = dx * dx + dy * dy; + if (d < minDist) { minDist = d; nearestId = id; } + }); + + if (nearestId) { + const display = sigma.getNodeDisplayData(nearestId); + if (display) { + sigma.getCamera().animate( + { x: display.x, y: display.y }, + { duration: 300 }, + ); + } + } + }, + [sigma, graph, size, getBounds], + ); + + if (graph.order === 0) return null; + + return ( + + ); +} diff --git a/ui/web/src/components/graph/sigma-graph-search.tsx b/ui/web/src/components/graph/sigma-graph-search.tsx new file mode 100644 index 00000000..f6260645 --- /dev/null +++ b/ui/web/src/components/graph/sigma-graph-search.tsx @@ -0,0 +1,162 @@ +import { useState, useRef, useCallback, useEffect } from "react"; +import type Sigma from "sigma"; +import type Graph from "graphology"; +import { Search, X } from "lucide-react"; +import { Input } from "@/components/ui/input"; + +interface SigmaGraphSearchProps { + sigma: Sigma | null; + graph: Graph; + onNodeSelect?: (nodeId: string | null) => void; + placeholder?: string; +} + +interface SearchResult { + id: string; + label: string; + color: string; +} + +export function SigmaGraphSearch({ sigma, graph, onNodeSelect, placeholder = "Search nodes..." }: SigmaGraphSearchProps) { + const [query, setQuery] = useState(""); + const [results, setResults] = useState([]); + const [open, setOpen] = useState(false); + const [activeIndex, setActiveIndex] = useState(0); + const inputRef = useRef(null); + const containerRef = useRef(null); + + // Search graph nodes by label + const handleSearch = useCallback( + (q: string) => { + setQuery(q); + if (!q.trim() || graph.order === 0) { + setResults([]); + setOpen(false); + return; + } + const lower = q.toLowerCase(); + const matches: SearchResult[] = []; + graph.forEachNode((node, attrs) => { + if (matches.length >= 10) return; + const label = (attrs.label as string) || ""; + if (label.toLowerCase().includes(lower)) { + matches.push({ id: node, label, color: attrs.color as string }); + } + }); + setResults(matches); + setOpen(matches.length > 0); + setActiveIndex(0); + }, + [graph], + ); + + // Select a result: highlight + camera zoom + const selectResult = useCallback( + (nodeId: string) => { + onNodeSelect?.(nodeId); + if (sigma && graph.hasNode(nodeId)) { + // Use getNodeDisplayData for correct stage coords (not raw graph coords). + // Only zoom in if currently zoomed out (don't force zoom level if already zoomed in). + const nodeDisplay = sigma.getNodeDisplayData(nodeId); + if (nodeDisplay) { + const currentRatio = sigma.getCamera().ratio; + sigma.getCamera().animate( + { x: nodeDisplay.x, y: nodeDisplay.y, ratio: Math.min(currentRatio, 0.5) }, + { duration: 400 }, + ); + } + } + setOpen(false); + setQuery(""); + }, + [sigma, graph, onNodeSelect], + ); + + // Keyboard navigation in dropdown + const handleKeyDown = useCallback( + (e: React.KeyboardEvent) => { + if (!open || results.length === 0) return; + if (e.key === "ArrowDown") { + e.preventDefault(); + setActiveIndex((i) => Math.min(i + 1, results.length - 1)); + } else if (e.key === "ArrowUp") { + e.preventDefault(); + setActiveIndex((i) => Math.max(i - 1, 0)); + } else if (e.key === "Enter") { + e.preventDefault(); + const r = results[activeIndex]; + if (r) selectResult(r.id); + } else if (e.key === "Escape") { + setOpen(false); + inputRef.current?.blur(); + } + }, + [open, results, activeIndex, selectResult], + ); + + // Close dropdown on outside click + useEffect(() => { + const handleClick = (e: MouseEvent) => { + if (containerRef.current && !containerRef.current.contains(e.target as Node)) { + setOpen(false); + } + }; + document.addEventListener("mousedown", handleClick); + return () => document.removeEventListener("mousedown", handleClick); + }, []); + + return ( + + + + handleSearch(e.target.value)} + onKeyDown={handleKeyDown} + onFocus={() => query && results.length > 0 && setOpen(true)} + placeholder={placeholder} + className="h-7 pl-7 pr-7 text-base md:text-xs" + role="combobox" + aria-expanded={open} + aria-autocomplete="list" + aria-controls="graph-search-listbox" + aria-label={placeholder} + /> + {query && ( + { setQuery(""); setResults([]); setOpen(false); }} + className="absolute right-1.5 top-1/2 -translate-y-1/2 p-0.5 rounded hover:bg-muted" + aria-label="Clear search" + > + + + )} + + + {/* Dropdown results */} + {open && ( + + {results.map((r, i) => ( + selectResult(r.id)} + className={`flex items-center gap-2 w-full px-2.5 py-1.5 text-xs text-left hover:bg-accent ${ + i === activeIndex ? "bg-accent" : "" + }`} + > + + {r.label} + + ))} + + )} + + ); +} diff --git a/ui/web/src/components/graph/use-sigma-keyboard.ts b/ui/web/src/components/graph/use-sigma-keyboard.ts new file mode 100644 index 00000000..c1f8c589 --- /dev/null +++ b/ui/web/src/components/graph/use-sigma-keyboard.ts @@ -0,0 +1,88 @@ +import { useEffect, type RefObject } from "react"; +import type Sigma from "sigma"; +import type Graph from "graphology"; + +interface UseSigmaKeyboardOptions { + sigma: Sigma | null; + graph: Graph; + /** Container element — shortcuts only fire when this element or its children are focused */ + containerRef: RefObject; + selectedNodeId?: string | null; + onNodeSelect?: (nodeId: string | null) => void; + /** Ref to search input — "/" focuses it */ + searchInputRef?: RefObject; +} + +/** + * Keyboard shortcuts for Sigma graph views. + * Only active when the graph container (or a child) is focused. + * + * | Key | Action | + * |---------|---------------------| + * | + / = | Zoom in | + * | - | Zoom out | + * | R | Reset view (fit all)| + * | F | Focus selected node | + * | Escape | Deselect all | + * | / | Focus search input | + */ +export function useSigmaKeyboard({ + sigma, + graph, + containerRef, + selectedNodeId, + onNodeSelect, + searchInputRef, +}: UseSigmaKeyboardOptions) { + useEffect(() => { + const container = containerRef.current; + if (!container || !sigma) return; + + const handler = (e: KeyboardEvent) => { + // Ignore if typing in an input/textarea + const tag = (e.target as HTMLElement).tagName; + if (tag === "INPUT" || tag === "TEXTAREA" || tag === "SELECT") return; + + switch (e.key) { + case "+": + case "=": + e.preventDefault(); + sigma.getCamera().animatedZoom({ duration: 200 }); + break; + case "-": + e.preventDefault(); + sigma.getCamera().animatedUnzoom({ duration: 200 }); + break; + case "r": + case "R": + e.preventDefault(); + sigma.getCamera().animatedReset({ duration: 300 }); + break; + case "f": + case "F": + if (selectedNodeId && graph.hasNode(selectedNodeId)) { + e.preventDefault(); + const nodeDisplay = sigma.getNodeDisplayData(selectedNodeId); + if (nodeDisplay) { + const currentRatio = sigma.getCamera().ratio; + sigma.getCamera().animate( + { x: nodeDisplay.x, y: nodeDisplay.y, ratio: Math.min(currentRatio, 0.5) }, + { duration: 400 }, + ); + } + } + break; + case "Escape": + onNodeSelect?.(null); + break; + case "/": + e.preventDefault(); + searchInputRef?.current?.focus(); + break; + } + }; + + container.addEventListener("keydown", handler); + return () => container.removeEventListener("keydown", handler); + }, [sigma, graph, containerRef, selectedNodeId, onNodeSelect, searchInputRef]); +} diff --git a/ui/web/src/components/layout/system-settings-constants.ts b/ui/web/src/components/layout/system-settings-constants.ts index 504310ea..3bb1226c 100644 --- a/ui/web/src/components/layout/system-settings-constants.ts +++ b/ui/web/src/components/layout/system-settings-constants.ts @@ -42,6 +42,8 @@ export interface InitState { kgProvider: string; kgModel: string; kgMinConfidence: string; + bgProvider: string; + bgModel: string; } export const DEFAULTS: InitState = { @@ -51,6 +53,7 @@ export const DEFAULTS: InitState = { compProvider: "", compModel: "", compThreshold: "", compKeepRecent: "", compMaxTokens: "", kgProvider: "", kgModel: "", kgMinConfidence: "0.75", + bgProvider: "", bgModel: "", }; export function parseBool(v: string | undefined, fallback: boolean): boolean { diff --git a/ui/web/src/components/layout/system-settings-modal.tsx b/ui/web/src/components/layout/system-settings-modal.tsx index ce36a27c..01cc4815 100644 --- a/ui/web/src/components/layout/system-settings-modal.tsx +++ b/ui/web/src/components/layout/system-settings-modal.tsx @@ -1,6 +1,6 @@ import { useState, useEffect, useCallback } from "react"; import { useTranslation } from "react-i18next"; -import { Settings2, Loader2, Save, AlertTriangle, Info, ExternalLink, Network } from "lucide-react"; +import { Settings2, Loader2, Save, AlertTriangle, Info, ExternalLink, Network, Cog } from "lucide-react"; import { Link } from "react-router"; import { Dialog, DialogContent, DialogHeader, DialogTitle } from "@/components/ui/dialog"; import { Button } from "@/components/ui/button"; @@ -57,6 +57,10 @@ export function SystemSettingsModal({ open, onOpenChange }: SystemSettingsModalP const [kgModel, setKgModel] = useState(""); const [kgMinConfidence, setKgMinConfidence] = useState("0.75"); + // Background Workers + const [bgProvider, setBgProvider] = useState(""); + const [bgModel, setBgModel] = useState(""); + const applyConfigs = useCallback(( configs: Record, kgSettings?: { extraction_provider?: string; extraction_model?: string; min_confidence?: number }, @@ -71,12 +75,14 @@ export function SystemSettingsModal({ open, onOpenChange }: SystemSettingsModalP compMaxTokens: configs["compaction.max_tokens"] ?? "", kgProvider: kgSettings?.extraction_provider ?? "", kgModel: kgSettings?.extraction_model ?? "", kgMinConfidence: String(kgSettings?.min_confidence ?? 0.75), + bgProvider: configs["background.provider"] ?? "", bgModel: configs["background.model"] ?? "", }; setInit(s); setEmbProvider(s.embProvider); setEmbModel(s.embModel); setEmbMaxChunkLen(s.embMaxChunkLen); setEmbChunkOverlap(s.embChunkOverlap); setToolStatus(s.toolStatus); setBlockReply(s.blockReply); setIntentClassify(s.intentClassify); setCompProvider(s.compProvider); setCompModel(s.compModel); setCompThreshold(s.compThreshold); setCompKeepRecent(s.compKeepRecent); setCompMaxTokens(s.compMaxTokens); setKgProvider(s.kgProvider); setKgModel(s.kgModel); setKgMinConfidence(s.kgMinConfidence); + setBgProvider(s.bgProvider); setBgModel(s.bgModel); resetEmb(); }, [resetEmb]); @@ -118,6 +124,8 @@ export function SystemSettingsModal({ open, onOpenChange }: SystemSettingsModalP if (compThreshold !== init.compThreshold) updates["compaction.threshold"] = compThreshold; if (compKeepRecent !== init.compKeepRecent) updates["compaction.keep_recent"] = compKeepRecent; if (compMaxTokens !== init.compMaxTokens) updates["compaction.max_tokens"] = compMaxTokens; + if (bgProvider !== init.bgProvider) updates["background.provider"] = bgProvider; + if (bgModel !== init.bgModel) updates["background.model"] = bgModel; for (const [key, value] of Object.entries(updates)) await http.put(`/v1/system-configs/${key}`, { value }); const kgChanged = kgProvider !== init.kgProvider || kgModel !== init.kgModel || kgMinConfidence !== init.kgMinConfidence; if (kgChanged) { @@ -179,6 +187,20 @@ export function SystemSettingsModal({ open, onOpenChange }: SystemSettingsModalP + {/* Background Workers */} + + + {t("bg.title")} + {t("bg.description")} + + + { setBgProvider(v); setBgModel(""); }} model={bgModel} onModelChange={setBgModel} allowEmpty providerLabel={t("bg.provider")} modelLabel={t("bg.model")} providerTip={t("bg.providerTip")} modelTip={t("bg.modelTip")} providerPlaceholder={t("bg.providerPlaceholder")} modelPlaceholder={t("bg.modelPlaceholder")} /> + + {t("bg.info")} + + + + c.value === configType)?.descKey ?? ""; return ( - + {/* Header */} @@ -305,6 +305,6 @@ export function AgentPermissionsTab({ agentId }: AgentPermissionsTabProps) { )} )} - + ); } diff --git a/ui/web/src/pages/agents/agent-detail/config-sections/memory-section.tsx b/ui/web/src/pages/agents/agent-detail/config-sections/memory-section.tsx index c8487f72..42d2ff61 100644 --- a/ui/web/src/pages/agents/agent-detail/config-sections/memory-section.tsx +++ b/ui/web/src/pages/agents/agent-detail/config-sections/memory-section.tsx @@ -1,7 +1,7 @@ import { useTranslation } from "react-i18next"; import { Input } from "@/components/ui/input"; import { Switch } from "@/components/ui/switch"; -import type { MemoryConfig } from "@/types/agent"; +import type { MemoryConfig, DreamingConfig } from "@/types/agent"; import { InfoLabel, numOrUndef } from "./config-section"; interface MemorySectionProps { @@ -12,6 +12,10 @@ interface MemorySectionProps { export function MemorySection({ value, onChange }: MemorySectionProps) { const { t } = useTranslation("agents"); const s = "configSections.memory"; + const ds = "configSections.dreaming"; + const dreaming: DreamingConfig = value.dreaming ?? {}; + const setDreaming = (patch: Partial) => + onChange({ ...value, dreaming: { ...dreaming, ...patch } }); return ( @@ -95,6 +99,51 @@ export function MemorySection({ value, onChange }: MemorySectionProps) { /> + {/* Phase 8 — Dreaming worker per-agent overrides. + Nested inside memory since dreaming is memory consolidation. + All fields optional; nil = use backend defaults. */} + + + {t(`${ds}.title`)} + {t(`${ds}.description`)} + + + setDreaming({ enabled: v })} + /> + {t(`${ds}.enabled`)} + + + + {t(`${ds}.threshold`)} + setDreaming({ threshold: numOrUndef(e.target.value) })} + className="text-base md:text-sm" + /> + + + {t(`${ds}.debounceMs`)} + setDreaming({ debounce_ms: numOrUndef(e.target.value) })} + className="text-base md:text-sm" + /> + + + + setDreaming({ verbose_log: v })} + /> + {t(`${ds}.verboseLog`)} + + ); diff --git a/ui/web/src/pages/agents/agent-detail/evolution-tab/evolution-metrics-charts.tsx b/ui/web/src/pages/agents/agent-detail/evolution-tab/evolution-metrics-charts.tsx index 821ee035..abedc644 100644 --- a/ui/web/src/pages/agents/agent-detail/evolution-tab/evolution-metrics-charts.tsx +++ b/ui/web/src/pages/agents/agent-detail/evolution-tab/evolution-metrics-charts.tsx @@ -18,16 +18,20 @@ export function EvolutionMetricsCharts({ toolAggs, retrievalAggs, loading }: Evo return ; } + // API returns rates as 0-1 fractions; convert to 0-100 for chart display. + const toolData = toolAggs.map((a) => ({ ...a, success_rate: a.success_rate * 100 })); + const retrievalData = retrievalAggs.map((a) => ({ ...a, usage_rate: a.usage_rate * 100 })); + return ( {/* Tool Success Rate */} {t("detail.evolution.toolSuccess")} - {toolAggs.length === 0 ? ( + {toolData.length === 0 ? ( {t("detail.evolution.noMetrics")} ) : ( - + `${v}%`} /> @@ -35,12 +39,12 @@ export function EvolutionMetricsCharts({ toolAggs, retrievalAggs, loading }: Evo formatter={(value, _name, props) => { const v = Number(value ?? 0); const p = props?.payload as ToolAggregate | undefined; - return [`${v.toFixed(1)}%`, p ? `${p.call_count} calls, ${p.avg_duration_ms.toFixed(0)}ms avg` : ""]; + return [`${v.toFixed(1)}%`, p ? t("detail.evolution.tooltipCalls", { count: p.call_count, ms: p.avg_duration_ms.toFixed(0) }) : ""]; }} /> {t("detail.evolution.retrievalQuality")} - {retrievalAggs.length === 0 ? ( + {retrievalData.length === 0 ? ( {t("detail.evolution.noMetrics")} ) : ( - + `${v}%`} /> @@ -65,12 +69,12 @@ export function EvolutionMetricsCharts({ toolAggs, retrievalAggs, loading }: Evo formatter={(value, _name, props) => { const v = Number(value ?? 0); const p = props?.payload as RetrievalAggregate | undefined; - return [`${v.toFixed(1)}%`, p ? `${p.query_count} queries, score ${p.avg_score.toFixed(2)}` : ""]; + return [`${v.toFixed(1)}%`, p ? t("detail.evolution.tooltipQueries", { count: p.query_count, score: p.avg_score.toFixed(2) }) : ""]; }} /> = { threshold: "bg-blue-100 text-blue-700 dark:bg-blue-900 dark:text-blue-300", tool_order: "bg-orange-100 text-orange-700 dark:bg-orange-900 dark:text-orange-300", skill_add: "bg-green-100 text-green-700 dark:bg-green-900 dark:text-green-300", - memory_prune: "bg-gray-100 text-gray-700 dark:bg-gray-800 dark:text-gray-300", }; const STATUS_COLORS: Record = { @@ -62,11 +61,11 @@ export function EvolutionSuggestionsTable({ suggestions, loading, onUpdateStatus - Type - Suggestion - Status - Created - Actions + {t("detail.evolution.colType")} + {t("detail.evolution.colSuggestion")} + {t("detail.evolution.colStatus")} + {t("detail.evolution.colCreated")} + {t("detail.evolution.colActions")} @@ -145,10 +144,10 @@ export function EvolutionSuggestionsTable({ suggestions, loading, onUpdateStatus setConfirm(null)} disabled={acting}> - Cancel + {t("detail.evolution.cancel")} - {acting ? "..." : "Confirm"} + {acting ? t("detail.evolution.confirming") : t("detail.evolution.confirm")} diff --git a/ui/web/src/pages/channels/channel-schemas.ts b/ui/web/src/pages/channels/channel-schemas.ts index ba92f452..3dc65d4d 100644 --- a/ui/web/src/pages/channels/channel-schemas.ts +++ b/ui/web/src/pages/channels/channel-schemas.ts @@ -69,6 +69,16 @@ export const credentialsSchema: Record = { ], zalo_personal: [], whatsapp: [], + facebook: [ + { key: "page_access_token", label: "Page Access Token", type: "password", required: true, help: "From Facebook Developer Console → Your App → Messenger → Page Access Token" }, + { key: "app_secret", label: "App Secret", type: "password", required: true, help: "From Facebook Developer Console → Your App → Settings → Basic" }, + { key: "verify_token", label: "Webhook Verify Token", type: "password", required: true, help: "A secret string you choose, used to verify the webhook URL" }, + ], + pancake: [ + { key: "api_key", label: "API Key", type: "password", required: true, help: "Pancake user-level API key from pages.fm account settings" }, + { key: "page_access_token", label: "Page Access Token", type: "password", required: true, help: "Page-level token from Pancake dashboard → Page Settings" }, + { key: "webhook_secret", label: "Webhook Secret (Optional)", type: "password", help: "HMAC-SHA256 secret for webhook signature verification. Leave empty to skip verification." }, + ], }; // --- Config schemas --- @@ -153,6 +163,27 @@ export const configSchema: Record = { { key: "allow_from", label: "Allowed Users", type: "tags", help: "WhatsApp user IDs" }, { key: "block_reply", label: "Block Reply", type: "select", options: blockReplyOptions, defaultValue: "inherit", help: "Deliver intermediate text during tool iterations" }, ], + facebook: [ + { key: "page_id", label: "Page ID", type: "text", required: true, help: "Facebook Page numeric ID" }, + { key: "features.comment_reply", label: "Comment Auto-Reply", type: "boolean", defaultValue: false }, + { key: "features.messenger_auto_reply", label: "Messenger Auto-Reply", type: "boolean", defaultValue: false }, + { key: "features.first_inbox", label: "First Inbox DM", type: "boolean", defaultValue: false, help: "Send a one-time DM to commenters after their first comment reply" }, + { key: "comment_reply_options.include_post_context", label: "Include Post Context", type: "boolean", defaultValue: false, help: "Fetch original post content for comment context" }, + { key: "comment_reply_options.max_thread_depth", label: "Max Thread Depth", type: "number", defaultValue: 10 }, + { key: "messenger_options.session_timeout", label: "Messenger Session Timeout", type: "text", placeholder: "e.g. 30m" }, + { key: "post_context_cache_ttl", label: "Post Cache TTL", type: "text", placeholder: "e.g. 15m" }, + { key: "first_inbox_message", label: "First Inbox DM Text", type: "textarea", help: "Custom DM sent to first-time commenters. Defaults to Vietnamese if empty." }, + { key: "allow_from", label: "Allowed Users", type: "tags", help: "Facebook user IDs" }, + { key: "block_reply", label: "Block Reply", type: "select", options: blockReplyOptions, defaultValue: "inherit" }, + ], + pancake: [ + { key: "page_id", label: "Page ID", type: "text", required: true, help: "Pancake page ID (numeric, from Pancake dashboard)" }, + { key: "platform", label: "Platform (auto-detected)", type: "text", placeholder: "Leave empty — resolved at startup", help: "facebook / zalo / instagram / tiktok / whatsapp / line. Auto-detected if empty." }, + { key: "features.inbox_reply", label: "Inbox Auto-Reply", type: "boolean", defaultValue: true }, + { key: "features.comment_reply", label: "Comment Reply", type: "boolean", defaultValue: false }, + { key: "allow_from", label: "Allowed Users", type: "tags", help: "Sender IDs to whitelist. Empty = accept all." }, + { key: "block_reply", label: "Block Reply", type: "select", options: blockReplyOptions, defaultValue: "inherit" }, + ], }; // --- Group override schema (Telegram per-group/topic overrides) --- diff --git a/ui/web/src/pages/chat/chat-page.tsx b/ui/web/src/pages/chat/chat-page.tsx index 36fbf612..5cf0ab5c 100644 --- a/ui/web/src/pages/chat/chat-page.tsx +++ b/ui/web/src/pages/chat/chat-page.tsx @@ -81,8 +81,8 @@ export function ChatPage() { const isOwn = !sessionKey || isOwnSession(sessionKey, userId); const handleMessageAdded = useCallback( - (msg: { role: "user" | "assistant" | "tool"; content: string; timestamp?: number }) => { - addLocalMessage(msg); + (msg: { role: "user" | "assistant" | "tool"; content: string; timestamp?: number }, key?: string) => { + addLocalMessage(msg, key); }, [addLocalMessage], ); @@ -234,7 +234,16 @@ export function ChatPage() { )} - setTaskPanelOpen((v) => !v)} taskPanelOpen={taskPanelOpen} /> + setTaskPanelOpen((v) => !v)} + taskPanelOpen={taskPanelOpen} + session={sessions.find((s) => s.key === sessionKey) ?? null} + /> {sendError && ( diff --git a/ui/web/src/pages/chat/hooks/use-chat-messages.ts b/ui/web/src/pages/chat/hooks/use-chat-messages.ts index 5f11f15a..f59fa420 100644 --- a/ui/web/src/pages/chat/hooks/use-chat-messages.ts +++ b/ui/web/src/pages/chat/hooks/use-chat-messages.ts @@ -7,6 +7,11 @@ import type { ChatMessage, AgentEventPayload, ToolStreamEntry, RunActivity, Acti import { toFileUrl, mediaKindFromMime } from "@/lib/file-helpers"; import { transformHistoryMessages } from "@/adapters/chat-message.adapter"; import { useChatTeamTasks } from "./use-chat-team-tasks"; +import { useChatMessagesStore } from "@/stores/use-chat-messages-store"; + +// Stable empty array — avoids creating a new reference on every render inside +// Zustand selectors, which would trigger an infinite re-render loop (React #185). +const EMPTY_MESSAGES: ChatMessage[] = []; /** * Manages chat message history and real-time streaming for a session. @@ -14,11 +19,19 @@ import { useChatTeamTasks } from "./use-chat-team-tasks"; */ export function useChatMessages(sessionKey: string, agentId: string) { const ws = useWs(); - const [messages, setMessages] = useState([]); - const [streamText, setStreamText] = useState(null); - const [thinkingText, setThinkingText] = useState(null); + const messages = useChatMessagesStore((s) => sessionKey ? (s.sessions[sessionKey]?.messages ?? EMPTY_MESSAGES) : EMPTY_MESSAGES); + const streamText = useChatMessagesStore((s) => sessionKey ? (s.sessions[sessionKey]?.streamText ?? null) : null); + const thinkingText = useChatMessagesStore((s) => sessionKey ? (s.sessions[sessionKey]?.thinkingText ?? null) : null); + const isRunning = useChatMessagesStore((s) => sessionKey ? (s.sessions[sessionKey]?.isRunning ?? false) : false); + + const setSessionMessages = useChatMessagesStore((s) => s.setSessionMessages); + const updateSessionMessages = useChatMessagesStore((s) => s.updateSessionMessages); + const setSessionStream = useChatMessagesStore((s) => s.setSessionStream); + const setSessionThinking = useChatMessagesStore((s) => s.setSessionThinking); + const setSessionRunning = useChatMessagesStore((s) => s.setSessionRunning); + + // Local state for non-persistent UI elements const [toolStream, setToolStream] = useState([]); - const [isRunning, setIsRunning] = useState(false); const [loading, setLoading] = useState(false); const [activity, setActivity] = useState(null); const [blockReplies, setBlockReplies] = useState([]); @@ -38,10 +51,15 @@ export function useChatMessages(sessionKey: string, agentId: string) { const rafPendingRef = useRef(false); const rafHandleRef = useRef(0); - // Add a local message optimistically - const addLocalMessage = useCallback((msg: ChatMessage) => { - setMessages((prev) => [...prev, msg]); - }, []); + // Add a local message optimistically. + // `key` is optional: callers that know the target session key (e.g. new-chat + // send flow, where the URL hasn't navigated yet) should pass it explicitly. + const addLocalMessage = useCallback((msg: ChatMessage, key?: string) => { + const targetKey = key ?? sessionKey; + if (targetKey) { + updateSessionMessages(targetKey, (prev) => [...prev, msg]); + } + }, [sessionKey, updateSessionMessages]); // Team task handling (extracted hook) const { teamTasks, setTeamTasks } = useChatTeamTasks(addLocalMessage); @@ -67,23 +85,32 @@ export function useChatMessages(sessionKey: string, agentId: string) { return; } - setStreamText(null); setThinkingText(null); setToolStream([]); - setIsRunning(false); setActivity(null); setBlockReplies([]); setTeamTasks([]); - runIdRef.current = null; expectingRunRef.current = false; - streamRef.current = ""; thinkingRef.current = ""; - toolStreamRef.current = []; activityRef.current = null; blockRepliesRef.current = []; - cancelAnimationFrame(rafHandleRef.current); rafPendingRef.current = false; - if (!sessionKey) setMessages([]); - }, [sessionKey, setTeamTasks]); + setSessionStream(sessionKey, null); + setSessionThinking(sessionKey, null); + setSessionRunning(sessionKey, false); + setToolStream([]); + setActivity(null); + setBlockReplies([]); + setTeamTasks([]); + runIdRef.current = null; + expectingRunRef.current = false; + streamRef.current = ""; + thinkingRef.current = ""; + toolStreamRef.current = []; + activityRef.current = null; + blockRepliesRef.current = []; + cancelAnimationFrame(rafHandleRef.current); + rafPendingRef.current = false; + }, [sessionKey, setTeamTasks, setSessionStream, setSessionThinking, setSessionRunning]); // Load history const loadHistory = useCallback(async (mediaItems?: MediaItem[]) => { if (!ws.isConnected || !sessionKey) { setLoading(false); return; } try { const res = await ws.call<{ messages: Message[] }>(Methods.CHAT_HISTORY, { agentId, sessionKey }); - setMessages(transformHistoryMessages(res.messages ?? [], mediaItems)); + setSessionMessages(sessionKey, transformHistoryMessages(res.messages ?? [], mediaItems)); } catch { /* will retry */ } finally { setLoading(false); } - }, [ws, agentId, sessionKey]); + }, [ws, agentId, sessionKey, setSessionMessages]); // Load history + restore running state when session changes useEffect(() => { @@ -99,7 +126,7 @@ export function useChatMessages(sessionKey: string, agentId: string) { ws.call<{ isRunning?: boolean; runId?: string; activity?: RunActivity }>(Methods.CHAT_SESSION_STATUS, { sessionKey }) .then((res) => { if (cancelled) return; - if (res.isRunning) { setIsRunning(true); if (res.runId) runIdRef.current = res.runId; } + if (res.isRunning) { setSessionRunning(sessionKey, true); if (res.runId) runIdRef.current = res.runId; } if (res.activity) { setActivity(res.activity); activityRef.current = res.activity; } }).catch((err) => console.error("[useChatMessages] session status failed:", err)); ws.call<{ tasks?: ActiveTeamTask[] }>(Methods.TEAMS_TASK_ACTIVE_BY_SESSION, { sessionKey }) @@ -107,7 +134,7 @@ export function useChatMessages(sessionKey: string, agentId: string) { .catch((err) => console.error("[useChatMessages] active tasks failed:", err)); } return () => { cancelled = true; }; - }, [sessionKey, loadHistory, ws, setTeamTasks]); + }, [sessionKey, loadHistory, ws, setTeamTasks, setSessionRunning]); // Called before sending so event handler captures run.started const expectRun = useCallback(() => { expectingRunRef.current = true; }, []); @@ -123,9 +150,15 @@ export function useChatMessages(sessionKey: string, agentId: string) { // Capture run.started if (event.type === "run.started" && event.agentId === agentIdRef.current) { if (expectingRunRef.current || event.runKind === "announce") { - runIdRef.current = event.runId; expectingRunRef.current = false; - setIsRunning(true); setStreamText(null); setThinkingText(null); setToolStream([]); - streamRef.current = ""; thinkingRef.current = ""; toolStreamRef.current = []; + runIdRef.current = event.runId; + expectingRunRef.current = false; + setSessionRunning(sessionKeyRef.current, true); + setSessionStream(sessionKeyRef.current, null); + setSessionThinking(sessionKeyRef.current, null); + setToolStream([]); + streamRef.current = ""; + thinkingRef.current = ""; + toolStreamRef.current = []; } return; } @@ -139,7 +172,8 @@ export function useChatMessages(sessionKey: string, agentId: string) { rafPendingRef.current = true; rafHandleRef.current = requestAnimationFrame(() => { rafPendingRef.current = false; - setThinkingText(thinkingRef.current); setStreamText(streamRef.current); + setSessionThinking(sessionKeyRef.current, thinkingRef.current); + setSessionStream(sessionKeyRef.current, streamRef.current); }); } break; @@ -150,7 +184,8 @@ export function useChatMessages(sessionKey: string, agentId: string) { rafPendingRef.current = true; rafHandleRef.current = requestAnimationFrame(() => { rafPendingRef.current = false; - setStreamText(streamRef.current); setThinkingText(thinkingRef.current); + setSessionStream(sessionKeyRef.current, streamRef.current); + setSessionThinking(sessionKeyRef.current, thinkingRef.current); }); } break; @@ -201,48 +236,69 @@ export function useChatMessages(sessionKey: string, agentId: string) { } case "run.completed": { cancelAnimationFrame(rafHandleRef.current); rafPendingRef.current = false; - setIsRunning(false); runIdRef.current = null; + setSessionRunning(sessionKeyRef.current, false); + runIdRef.current = null; const hadTools = toolStreamRef.current.length > 0; const streamed = streamRef.current; const thinking = thinkingRef.current || undefined; - setStreamText(null); setThinkingText(null); setToolStream([]); - streamRef.current = ""; thinkingRef.current = ""; toolStreamRef.current = []; - activityRef.current = null; setActivity(null); - blockRepliesRef.current = []; setBlockReplies([]); + setSessionStream(sessionKeyRef.current, null); + setSessionThinking(sessionKeyRef.current, null); + setToolStream([]); + streamRef.current = ""; + thinkingRef.current = ""; + toolStreamRef.current = []; + activityRef.current = null; + setActivity(null); + blockRepliesRef.current = []; + setBlockReplies([]); const rawMedia = event.payload?.media; const mediaItems: MediaItem[] | undefined = rawMedia?.length ? rawMedia.map((m) => ({ path: toFileUrl(m.path), mimeType: m.content_type ?? "application/octet-stream", fileName: m.path.split("?")[0]?.split("/").pop() ?? "file", size: m.size, kind: mediaKindFromMime(m.content_type ?? "") })) : undefined; if (streamed && !hadTools) { - setMessages((prev) => [...prev, { role: "assistant", content: streamed, thinking, timestamp: Date.now(), mediaItems }]); + updateSessionMessages(sessionKeyRef.current, (prev) => [...prev, { role: "assistant", content: streamed, thinking, timestamp: Date.now(), mediaItems }]); } else { loadHistory(mediaItems); } break; } case "run.failed": { cancelAnimationFrame(rafHandleRef.current); rafPendingRef.current = false; - setIsRunning(false); runIdRef.current = null; - setStreamText(null); setThinkingText(null); setToolStream([]); - streamRef.current = ""; thinkingRef.current = ""; - activityRef.current = null; setActivity(null); - blockRepliesRef.current = []; setBlockReplies([]); - setMessages((prev) => [...prev, { role: "assistant", content: `Error: ${event.payload?.error ?? "Unknown error"}`, timestamp: Date.now() }]); + setSessionRunning(sessionKeyRef.current, false); + runIdRef.current = null; + setSessionStream(sessionKeyRef.current, null); + setSessionThinking(sessionKeyRef.current, null); + setToolStream([]); + streamRef.current = ""; + thinkingRef.current = ""; + activityRef.current = null; + setActivity(null); + blockRepliesRef.current = []; + setBlockReplies([]); + updateSessionMessages(sessionKeyRef.current, (prev) => [...prev, { role: "assistant", content: `Error: ${event.payload?.error ?? "Unknown error"}`, timestamp: Date.now() }]); break; } case "run.cancelled": { cancelAnimationFrame(rafHandleRef.current); rafPendingRef.current = false; - setIsRunning(false); runIdRef.current = null; + setSessionRunning(sessionKeyRef.current, false); + runIdRef.current = null; const streamed = streamRef.current; - setStreamText(null); setThinkingText(null); setToolStream([]); - streamRef.current = ""; thinkingRef.current = ""; toolStreamRef.current = []; - activityRef.current = null; setActivity(null); - blockRepliesRef.current = []; setBlockReplies([]); - if (streamed) { setMessages((prev) => [...prev, { role: "assistant", content: streamed, timestamp: Date.now() }]); } - else { loadHistory(); } + setSessionStream(sessionKeyRef.current, null); + setSessionThinking(sessionKeyRef.current, null); + setToolStream([]); + streamRef.current = ""; + thinkingRef.current = ""; + toolStreamRef.current = []; + activityRef.current = null; + setActivity(null); + blockRepliesRef.current = []; + setBlockReplies([]); + if (streamed) { + updateSessionMessages(sessionKeyRef.current, (prev) => [...prev, { role: "assistant", content: streamed, timestamp: Date.now() }]); + } else { loadHistory(); } break; } } }, - [loadHistory], + [loadHistory, setSessionRunning, setSessionStream, setSessionThinking, updateSessionMessages], ); useWsEvent(Events.AGENT, handleAgentEvent); diff --git a/ui/web/src/pages/chat/hooks/use-chat-send.ts b/ui/web/src/pages/chat/hooks/use-chat-send.ts index 5bdc2d43..2e54d6ef 100644 --- a/ui/web/src/pages/chat/hooks/use-chat-send.ts +++ b/ui/web/src/pages/chat/hooks/use-chat-send.ts @@ -7,7 +7,7 @@ import type { AttachedFile } from "@/components/chat/chat-input"; interface UseChatSendOptions { agentId: string; - onMessageAdded: (msg: ChatMessage) => void; + onMessageAdded: (msg: ChatMessage, sessionKey?: string) => void; onExpectRun: () => void; } @@ -58,7 +58,7 @@ export function useChatSend({ role: "user", content: displayContent, timestamp: Date.now(), - }); + }, sessionKey); try { // Upload files first, then pass path+filename to chat.send diff --git a/ui/web/src/pages/memory/knowledge-graph/kg-entity-detail-dialog.tsx b/ui/web/src/pages/memory/knowledge-graph/kg-entity-detail-dialog.tsx index ad7d70e1..8bf85358 100644 --- a/ui/web/src/pages/memory/knowledge-graph/kg-entity-detail-dialog.tsx +++ b/ui/web/src/pages/memory/knowledge-graph/kg-entity-detail-dialog.tsx @@ -239,7 +239,7 @@ export function KGEntityDetailDialog({ open, onOpenChange, agentId, entity, getE ) : graphData ? ( - + ) : ( diff --git a/ui/web/src/pages/memory/knowledge-graph/kg-graph-view.tsx b/ui/web/src/pages/memory/knowledge-graph/kg-graph-view.tsx index 8b4e1204..283725a7 100644 --- a/ui/web/src/pages/memory/knowledge-graph/kg-graph-view.tsx +++ b/ui/web/src/pages/memory/knowledge-graph/kg-graph-view.tsx @@ -1,286 +1,164 @@ -import { useRef, useMemo, useCallback, useState, useEffect, useLayoutEffect } from "react"; -import ForceGraph2D, { type ForceGraphMethods } from "react-force-graph-2d"; -import { Maximize2, ZoomIn, ZoomOut } from "lucide-react"; -import { Button } from "@/components/ui/button"; +import { useRef, useMemo, useState, useCallback } from "react"; +import type Sigma from "sigma"; import { useTranslation } from "react-i18next"; -import { useUiStore } from "@/stores/use-ui-store"; import type { KGEntity, KGRelation } from "@/types/knowledge-graph"; -import { buildGraphData, limitEntitiesByDegree } from "@/adapters/kg-graph.adapter"; -import type { GraphNode, GraphLink } from "@/adapters/kg-graph.adapter"; +import { buildKGGraph, limitEntitiesByDegree, KG_TYPE_COLORS } from "@/adapters/kg-graph.adapter"; +import { SigmaGraphContainer } from "@/components/graph/sigma-graph-container"; +import { SigmaGraphControls } from "@/components/graph/sigma-graph-controls"; +import { SigmaGraphSearch } from "@/components/graph/sigma-graph-search"; +import { SigmaGraphFilters } from "@/components/graph/sigma-graph-filters"; +import { SigmaGraphMinimap } from "@/components/graph/sigma-graph-minimap"; +import { SigmaGraphKeyboardHelp } from "@/components/graph/sigma-graph-keyboard-help"; +import { useSigmaKeyboard } from "@/components/graph/use-sigma-keyboard"; -const NODE_LIMIT_OPTIONS = [100, 200, 300, 500] as const; const DEFAULT_NODE_LIMIT = 200; -const NODE_R = 5; -const DOUBLE_CLICK_MS = 280; interface KGGraphViewProps { entities: KGEntity[]; relations: KGRelation[]; onEntityClick?: (entity: KGEntity) => void; + /** Compact mode for embedded mini-graphs (entity detail dialog) */ + compact?: boolean; } -export function KGGraphView({ entities: allEntities, relations: allRelations, onEntityClick }: KGGraphViewProps) { +export function KGGraphView({ entities: allEntities, relations: allRelations, onEntityClick, compact = false }: KGGraphViewProps) { const { t } = useTranslation("memory"); - const theme = useUiStore((s) => s.theme); - const isDark = theme === "dark" || (theme === "system" && window.matchMedia("(prefers-color-scheme: dark)").matches); - - const graphRef = useRef>(undefined); const containerRef = useRef(null); - const [dimensions, setDimensions] = useState({ width: 600, height: 400 }); + const searchInputRef = useRef(null); + const [sigma, setSigma] = useState(null); const [selectedNodeId, setSelectedNodeId] = useState(null); const [nodeLimit, setNodeLimit] = useState(DEFAULT_NODE_LIMIT); - const [ready, setReady] = useState(false); - const [zoomLevel, setZoomLevel] = useState(1); + const [hiddenTypes, setHiddenTypes] = useState>(new Set()); + const [filtersOpen, setFiltersOpen] = useState(false); - const ZOOM_STEP = 1.5; - - const handleZoomIn = useCallback(() => { - const fg = graphRef.current; - if (!fg) return; - fg.zoom(Math.min((fg.zoom() ?? 1) * ZOOM_STEP, 8), 300); - }, []); - - const handleZoomOut = useCallback(() => { - const fg = graphRef.current; - if (!fg) return; - fg.zoom(Math.max((fg.zoom() ?? 1) / ZOOM_STEP, 0.1), 300); - }, []); - - // Double-click detection refs - const clickTimerRef = useRef | null>(null); - const lastClickRef = useRef(null); - - // --- Container sizing --- - // 1. Synchronous initial read before first paint to avoid wrong-size flash - useLayoutEffect(() => { - const el = containerRef.current; - if (el && el.clientWidth > 0 && el.clientHeight > 0) { - setDimensions({ width: el.clientWidth, height: el.clientHeight }); - setReady(true); - } - }, []); - - // 2. ResizeObserver for ongoing size changes (window resize, sidebar toggle) - useEffect(() => { - const el = containerRef.current; - if (!el) return; - const ro = new ResizeObserver(([entry]) => { - if (!entry) return; - const { width, height } = entry.contentRect; - if (width > 0 && height > 0) { - setDimensions({ width, height }); - setReady(true); - } - }); - ro.observe(el); - return () => ro.disconnect(); - }, []); - - // --- Limit entities by degree centrality --- const totalCount = allEntities.length; const isLimited = totalCount > nodeLimit; const entities = useMemo( () => limitEntitiesByDegree(allEntities, allRelations, nodeLimit), [allEntities, allRelations, nodeLimit], ); - const entityMap = useMemo(() => new Map(entities.map((e) => [e.id, e])), [entities]); + const graph = useMemo(() => buildKGGraph(entities, allRelations), [entities, allRelations]); - // --- Build graph data for react-force-graph --- - const graphData = useMemo( - () => buildGraphData(entities, allRelations), - [entities, allRelations], - ); + const handleNodeDoubleClick = useCallback((nodeId: string) => { + const entity = entityMap.get(nodeId); + if (entity) onEntityClick?.(entity); + }, [entityMap, onEntityClick]); - // --- Highlight sets derived from selected node --- - const { highlightNodes, highlightLinks } = useMemo(() => { - if (!selectedNodeId) return { highlightNodes: new Set(), highlightLinks: new Set() }; - const node = graphData.nodes.find((n) => n.id === selectedNodeId); - if (!node) return { highlightNodes: new Set(), highlightLinks: new Set() }; - return { - highlightNodes: new Set([selectedNodeId, ...node.neighbors]), - highlightLinks: new Set(node.linkIds), - }; - }, [selectedNodeId, graphData.nodes]); - - // --- Node Canvas rendering: colored circle + name label --- - const nodeCanvasObject = useCallback( - (node: GraphNode, ctx: CanvasRenderingContext2D, globalScale: number) => { - const x = node.x ?? 0; - const y = node.y ?? 0; - const isSelected = node.id === selectedNodeId; - const isNeighbor = highlightNodes.has(node.id); - const isDimmed = !!selectedNodeId && !isSelected && !isNeighbor; - const r = NODE_R + (node.degree >= 4 ? 2 : 0); - - // Circle fill - ctx.beginPath(); - ctx.arc(x, y, r, 0, 2 * Math.PI); - ctx.fillStyle = isDimmed ? `${node.color}30` : node.color; - ctx.fill(); - - // Selection glow ring - if (isSelected) { - ctx.strokeStyle = node.color; - ctx.lineWidth = 2.5 / globalScale; - ctx.shadowColor = node.color; - ctx.shadowBlur = 8 / globalScale; - ctx.stroke(); - ctx.shadowBlur = 0; - } else if (isNeighbor) { - ctx.strokeStyle = node.color; - ctx.lineWidth = 1.5 / globalScale; - ctx.stroke(); - } - - // Name label — show when zoomed enough or highlighted - if (globalScale > 0.5 || isSelected || isNeighbor) { - const fontSize = Math.max(11 / globalScale, 2); - ctx.font = `${isSelected ? "bold " : ""}${fontSize}px Inter, system-ui, sans-serif`; - ctx.textAlign = "center"; - ctx.textBaseline = "top"; - ctx.fillStyle = isDimmed - ? (isDark ? "rgba(255,255,255,0.12)" : "rgba(0,0,0,0.12)") - : (isDark ? "#e2e8f0" : "#1e293b"); - ctx.fillText(node.name, x, y + r + 2 / globalScale); - } - }, - [selectedNodeId, highlightNodes, isDark], - ); - - // --- Link canvas overlay: draw label text only for highlighted links --- - const linkCanvasObject = useCallback( - (link: any, ctx: CanvasRenderingContext2D, globalScale: number) => { - if (!highlightLinks.has(link.id)) return; - const src = link.source; - const tgt = link.target; - if (!src?.x || !tgt?.x) return; - const midX = (src.x + tgt.x) / 2; - const midY = (src.y + tgt.y) / 2; - const fontSize = Math.max(9 / globalScale, 2); - ctx.font = `${fontSize}px Inter, system-ui, sans-serif`; - ctx.textAlign = "center"; - ctx.textBaseline = "middle"; - // Background pill for readability - const text = link.label || ""; - const metrics = ctx.measureText(text); - const pad = 2 / globalScale; - ctx.fillStyle = isDark ? "rgba(15,23,42,0.85)" : "rgba(255,255,255,0.9)"; - ctx.fillRect(midX - metrics.width / 2 - pad, midY - fontSize / 2 - pad, metrics.width + pad * 2, fontSize + pad * 2); - // Text - ctx.fillStyle = isDark ? "#94a3b8" : "#64748b"; - ctx.fillText(text, midX, midY); - }, - [highlightLinks, isDark], - ); - - // --- Click handlers: single (highlight) vs double (open detail) --- - const handleNodeClick = useCallback( - (node: GraphNode) => { - if (clickTimerRef.current && lastClickRef.current === node.id) { - // Double click → open detail modal - clearTimeout(clickTimerRef.current); - clickTimerRef.current = null; - lastClickRef.current = null; - const entity = entityMap.get(node.id); - if (entity && onEntityClick) onEntityClick(entity); - return; - } - // Single click → highlight (after delay to distinguish from double) - if (clickTimerRef.current) clearTimeout(clickTimerRef.current); - lastClickRef.current = node.id; - clickTimerRef.current = setTimeout(() => { - setSelectedNodeId((prev) => (prev === node.id ? null : node.id)); - clickTimerRef.current = null; - }, DOUBLE_CLICK_MS); - }, - [entityMap, onEntityClick], - ); - - const handleBackgroundClick = useCallback(() => { - setSelectedNodeId(null); - }, []); - - const handleEngineStop = useCallback(() => { - // No auto zoom — initial view from warmupTicks is already good - }, []); + useSigmaKeyboard({ + sigma: compact ? null : sigma, + graph, + containerRef, + selectedNodeId, + onNodeSelect: setSelectedNodeId, + searchInputRef, + }); const hasEntities = allEntities.length > 0; + // Compact mode: simple graph only + if (compact) { + return ( + + + {!hasEntities ? ( + + {t("kg.graphView.empty")} + + ) : ( + + )} + + + ); + } + return ( - - + + {/* Top bar — responsive: legend stacks on mobile */} + {hasEntities && ( + + {/* KG type legend */} + + {Object.entries(KG_TYPE_COLORS).map(([type, color]) => ( + + + {type} + + ))} + + + + setFiltersOpen(!c)} + /> + + + + )} + + {/* Graph canvas + minimap overlay */} + {!hasEntities ? ( - {t("kg.graphView.empty")} - ) : ready ? "replace"} - nodePointerAreaPaint={(node: GraphNode, color, ctx) => { - const r = NODE_R + (node.degree >= 4 ? 2 : 0) + 3; - ctx.beginPath(); - ctx.arc(node.x ?? 0, node.y ?? 0, r, 0, 2 * Math.PI); - ctx.fillStyle = color; - ctx.fill(); - }} - linkColor={(link: any) => - highlightLinks.has(link.id) ? "#64748b" : (isDark ? "#334155" : "#cbd5e1") - } - linkWidth={(link: any) => (highlightLinks.has(link.id) ? 2 : 0.5)} - linkCanvasObject={linkCanvasObject} - linkCanvasObjectMode={(link: any) => (highlightLinks.has(link.id) ? "after" : undefined)} - linkDirectionalParticles={(link: any) => (highlightLinks.has(link.id) ? 2 : 0)} - linkDirectionalParticleWidth={3} - linkDirectionalParticleSpeed={0.004} - onNodeClick={handleNodeClick} - onBackgroundClick={handleBackgroundClick} - onZoom={(transform) => setZoomLevel(transform.k)} - onEngineStop={handleEngineStop} - backgroundColor="transparent" - d3AlphaDecay={0.04} - d3VelocityDecay={0.3} - warmupTicks={40} - cooldownTime={4000} - enableNodeDrag={true} - minZoom={0.1} - maxZoom={8} - /> : null} + + {t("kg.graphView.empty")} + + ) : ( + <> + + + + + > + )} {/* Stats bar */} - - {t("kg.graphView.nodes", { count: totalCount })} - {t("kg.graphView.edges", { count: allRelations.length })} - {isLimited && ( - · {t("kg.graphView.limitNote", { limit: nodeLimit, total: totalCount })} - )} - - - - - - {Math.round(zoomLevel * 100)}% - - - - - setNodeLimit(Number(e.target.value))} - className="h-5 rounded border bg-background px-1 text-2xs" - > - {NODE_LIMIT_OPTIONS.map((n) => ( - {n} nodes - ))} - - graphRef.current?.zoomToFit(300, 20)}> - - - + ); } diff --git a/ui/web/src/pages/vault/hooks/use-enrichment-progress.ts b/ui/web/src/pages/vault/hooks/use-enrichment-progress.ts new file mode 100644 index 00000000..6f85794d --- /dev/null +++ b/ui/web/src/pages/vault/hooks/use-enrichment-progress.ts @@ -0,0 +1,27 @@ +import { useState } from "react"; +import { useWsEvent } from "@/hooks/use-ws-event"; + +export interface EnrichmentEvent { + phase: string; // enriching, complete + done: number; + total: number; + running: boolean; +} + +/** + * Listens to vault.enrich.progress WS events and returns current enrichment state. + * Progress auto-clears 3s after completion. + */ +export function useEnrichmentProgress() { + const [event, setEvent] = useState(null); + + useWsEvent("vault.enrich.progress", (payload) => { + const data = payload as EnrichmentEvent; + setEvent(data); + if (!data.running) { + setTimeout(() => setEvent(null), 3000); + } + }); + + return event; +} diff --git a/ui/web/src/pages/vault/hooks/use-vault-mutations.ts b/ui/web/src/pages/vault/hooks/use-vault-mutations.ts index 9f00687b..3773344b 100644 --- a/ui/web/src/pages/vault/hooks/use-vault-mutations.ts +++ b/ui/web/src/pages/vault/hooks/use-vault-mutations.ts @@ -1,21 +1,33 @@ import { useCallback } from "react"; +import { useState } from "react"; import { useQueryClient } from "@tanstack/react-query"; import { useHttp } from "@/hooks/use-ws"; import { toast } from "@/stores/use-toast-store"; import i18n from "@/i18n"; import type { VaultDocument, VaultLink } from "@/types/vault"; +interface RescanResult { + scanned: number; + new: number; + updated: number; + unchanged: number; + skipped: number; + errors: number; + truncated: boolean; +} + const VAULT_KEY = "vault"; -/** Create a new vault document. */ +/** Create a new vault document. Agent-scoped or cross-agent (empty agentId). */ export function useCreateDocument(agentId: string) { const http = useHttp(); const queryClient = useQueryClient(); const create = useCallback( - async (body: { path: string; title: string; doc_type: string; scope: string; metadata?: Record }) => { + async (body: { path: string; title: string; doc_type: string; scope: string; team_id?: string; metadata?: Record }) => { try { - const doc = await http.post(`/v1/agents/${agentId}/vault/documents`, body); + const url = agentId ? `/v1/agents/${agentId}/vault/documents` : `/v1/vault/documents`; + const doc = await http.post(url, body); await queryClient.invalidateQueries({ queryKey: [VAULT_KEY] }); toast.success(i18n.t("vault:toast.docCreated")); return doc; @@ -31,14 +43,14 @@ export function useCreateDocument(agentId: string) { } /** Update a vault document. */ -export function useUpdateDocument(agentId: string, docId: string) { +export function useUpdateDocument(docId: string) { const http = useHttp(); const queryClient = useQueryClient(); const update = useCallback( async (body: { title?: string; doc_type?: string; scope?: string; metadata?: Record }) => { try { - const doc = await http.put(`/v1/agents/${agentId}/vault/documents/${docId}`, body); + const doc = await http.put(`/v1/vault/documents/${docId}`, body); await queryClient.invalidateQueries({ queryKey: [VAULT_KEY] }); toast.success(i18n.t("vault:toast.docUpdated")); return doc; @@ -47,40 +59,40 @@ export function useUpdateDocument(agentId: string, docId: string) { throw err; } }, - [http, agentId, docId, queryClient], + [http, docId, queryClient], ); return { update }; } /** Delete a vault document. */ -export function useDeleteDocument(agentId: string, docId: string) { +export function useDeleteDocument(docId: string) { const http = useHttp(); const queryClient = useQueryClient(); const remove = useCallback(async () => { try { - await http.delete(`/v1/agents/${agentId}/vault/documents/${docId}`); + await http.delete(`/v1/vault/documents/${docId}`); await queryClient.invalidateQueries({ queryKey: [VAULT_KEY] }); toast.success(i18n.t("vault:toast.docDeleted")); } catch (err) { toast.error(i18n.t("vault:toast.docDeleteFailed"), err instanceof Error ? err.message : ""); throw err; } - }, [http, agentId, docId, queryClient]); + }, [http, docId, queryClient]); return { remove }; } /** Create a link between two vault documents. */ -export function useCreateLink(agentId: string) { +export function useCreateLink() { const http = useHttp(); const queryClient = useQueryClient(); const create = useCallback( async (body: { from_doc_id: string; to_doc_id: string; link_type: string; context?: string }) => { try { - const link = await http.post(`/v1/agents/${agentId}/vault/links`, body); + const link = await http.post(`/v1/vault/links`, body); await queryClient.invalidateQueries({ queryKey: [VAULT_KEY, "links"] }); await queryClient.invalidateQueries({ queryKey: [VAULT_KEY, "all-links"] }); toast.success(i18n.t("vault:toast.linkCreated")); @@ -90,20 +102,57 @@ export function useCreateLink(agentId: string) { throw err; } }, - [http, agentId, queryClient], + [http, queryClient], ); return { create }; } +/** Rescan workspace to sync vault documents from filesystem (tenant-wide). */ +export function useRescanWorkspace() { + const http = useHttp(); + const queryClient = useQueryClient(); + const [isPending, setIsPending] = useState(false); + + const rescan = useCallback(async () => { + setIsPending(true); + try { + const result = await http.post(`/v1/vault/rescan`, {}); + await queryClient.invalidateQueries({ queryKey: [VAULT_KEY] }); + + const parts: string[] = []; + if (result.new > 0) parts.push(i18n.t("vault:rescanNew", { count: result.new })); + if (result.updated > 0) parts.push(i18n.t("vault:rescanUpdated", { count: result.updated })); + if (result.unchanged > 0) parts.push(i18n.t("vault:rescanUnchanged", { count: result.unchanged })); + + const title = parts.length > 0 ? parts.join(", ") : i18n.t("vault:rescanNoFiles"); + const desc = result.truncated ? i18n.t("vault:rescanTruncated") : undefined; + toast.success(title, desc); + return result; + } catch (err) { + const status = (err as { status?: number })?.status; + if (status === 409) { + toast.warning(i18n.t("vault:rescanBusy")); + } else { + toast.error(i18n.t("vault:rescanError"), err instanceof Error ? err.message : ""); + } + throw err; + } finally { + setIsPending(false); + } + }, [http, queryClient]); + + return { rescan, isPending }; +} + /** Delete a vault link. */ -export function useDeleteLink(agentId: string, linkId: string) { +export function useDeleteLink(linkId: string) { const http = useHttp(); const queryClient = useQueryClient(); const remove = useCallback(async () => { try { - await http.delete(`/v1/agents/${agentId}/vault/links/${linkId}`); + await http.delete(`/v1/vault/links/${linkId}`); await queryClient.invalidateQueries({ queryKey: [VAULT_KEY, "links"] }); await queryClient.invalidateQueries({ queryKey: [VAULT_KEY, "all-links"] }); toast.success(i18n.t("vault:toast.linkDeleted")); @@ -111,7 +160,7 @@ export function useDeleteLink(agentId: string, linkId: string) { toast.error(i18n.t("vault:toast.linkDeleteFailed"), err instanceof Error ? err.message : ""); throw err; } - }, [http, agentId, linkId, queryClient]); + }, [http, linkId, queryClient]); return { remove }; } diff --git a/ui/web/src/pages/vault/hooks/use-vault-upload.ts b/ui/web/src/pages/vault/hooks/use-vault-upload.ts new file mode 100644 index 00000000..f98d1892 --- /dev/null +++ b/ui/web/src/pages/vault/hooks/use-vault-upload.ts @@ -0,0 +1,45 @@ +import { useCallback, useState } from "react"; +import { useQueryClient } from "@tanstack/react-query"; +import { useHttp } from "@/hooks/use-ws"; +import { toast } from "@/stores/use-toast-store"; +import i18n from "@/i18n"; +import type { VaultDocument } from "@/types/vault"; + +const VAULT_KEY = "vault"; + +interface UploadResult { + documents: Array<{ document?: VaultDocument; error?: string }>; + count: number; +} + +/** Upload files to vault via multipart POST /v1/vault/upload. */ +export function useVaultUpload() { + const http = useHttp(); + const queryClient = useQueryClient(); + const [isPending, setIsPending] = useState(false); + + const upload = useCallback( + async (files: File[], opts?: { agentId?: string; teamId?: string }) => { + setIsPending(true); + try { + const form = new FormData(); + if (opts?.agentId) form.append("agent_id", opts.agentId); + if (opts?.teamId) form.append("team_id", opts.teamId); + for (const f of files) form.append("files", f); + + const result = await http.upload("/v1/vault/upload", form); + await queryClient.invalidateQueries({ queryKey: [VAULT_KEY] }); + toast.success(i18n.t("vault:toast.uploadSuccess", { count: result.count })); + return result; + } catch (err) { + toast.error(i18n.t("vault:toast.uploadFailed"), err instanceof Error ? err.message : ""); + throw err; + } finally { + setIsPending(false); + } + }, + [http, queryClient], + ); + + return { upload, isPending }; +} diff --git a/ui/web/src/pages/vault/hooks/use-vault.ts b/ui/web/src/pages/vault/hooks/use-vault.ts index 154a2819..7b72c323 100644 --- a/ui/web/src/pages/vault/hooks/use-vault.ts +++ b/ui/web/src/pages/vault/hooks/use-vault.ts @@ -1,4 +1,4 @@ -import { useCallback, useMemo } from "react"; +import { useCallback, useEffect, useMemo, useState } from "react"; import { useQuery, useQueryClient } from "@tanstack/react-query"; import { useHttp } from "@/hooks/use-ws"; import type { VaultDocument, VaultLink, VaultSearchResult } from "@/types/vault"; @@ -72,39 +72,35 @@ export function useVaultSearch(agentId: string) { return { search, invalidate }; } -/** Fetch all links for a set of vault documents (for graph view). */ -export function useVaultAllLinks(agentId: string, documents: { id: string }[]) { +/** Tenant-wide vault search (agent_id optional). */ +export function useVaultSearchAll() { const http = useHttp(); + const search = useCallback( + async (query: string, opts?: { agentId?: string; docTypes?: string[]; teamId?: string; maxResults?: number }) => { + return http.post("/v1/vault/search", { + query, + agent_id: opts?.agentId || undefined, + doc_types: opts?.docTypes, + team_id: opts?.teamId || undefined, + max_results: opts?.maxResults ?? 20, + }); + }, + [http], + ); + + return { search }; +} + +/** Fetch all links for a set of vault documents (single batch query). */ +export function useVaultAllLinks(agentId: string, documents: { id: string }[]) { + const http = useHttp(); const docIds = useMemo(() => documents.map((d) => d.id), [documents]); const { data, isLoading } = useQuery({ queryKey: [VAULT_KEY, "all-links", agentId, [...docIds].sort().join(",")], - queryFn: async () => { - if (docIds.length === 0) return []; - const allLinks: VaultLink[] = []; - const batchSize = 10; - for (let i = 0; i < docIds.length; i += batchSize) { - const batch = docIds.slice(i, i + batchSize); - const results = await Promise.all( - batch.map((id) => - http.get<{ outlinks: VaultLink[]; backlinks: VaultLink[] }>( - `/v1/agents/${agentId}/vault/documents/${id}/links`, - ).catch(() => ({ outlinks: [], backlinks: [] })), - ), - ); - for (const r of results) { - allLinks.push(...r.outlinks); - } - } - const seen = new Set(); - return allLinks.filter((l) => { - if (seen.has(l.id)) return false; - seen.add(l.id); - return true; - }); - }, - enabled: !!agentId && docIds.length > 0, + queryFn: () => http.post("/v1/vault/links/batch", { doc_ids: docIds }), + enabled: docIds.length > 0, staleTime: 60_000, }); @@ -137,43 +133,11 @@ export function useVaultGraphData(agentId: string, opts?: { teamId?: string }) { [documents], ); - // Fetch links grouped by agent_id — works for both single-agent and all-agents mode. + // Fetch all links in one batch query (single SQL, no N+1). + const docIds = useMemo(() => documents.map((d) => d.id), [documents]); const { data: linksData, isLoading: linksLoading } = useQuery({ queryKey: [VAULT_KEY, "graph-links", docIdKey], - queryFn: async () => { - if (documents.length === 0) return []; - // Group doc IDs by agent_id - const byAgent = new Map(); - for (const doc of documents) { - const aid = doc.agent_id; - if (!aid) continue; - if (!byAgent.has(aid)) byAgent.set(aid, []); - byAgent.get(aid)!.push(doc.id); - } - - const allLinks: VaultLink[] = []; - for (const [aid, ids] of byAgent) { - const batchSize = 10; - for (let i = 0; i < ids.length; i += batchSize) { - const batch = ids.slice(i, i + batchSize); - const results = await Promise.all( - batch.map((id) => - http.get<{ outlinks: VaultLink[]; backlinks: VaultLink[] }>( - `/v1/agents/${aid}/vault/documents/${id}/links`, - ).catch(() => ({ outlinks: [], backlinks: [] })), - ), - ); - for (const r of results) allLinks.push(...r.outlinks); - } - } - // Dedup - const seen = new Set(); - return allLinks.filter((l) => { - if (seen.has(l.id)) return false; - seen.add(l.id); - return true; - }); - }, + queryFn: () => http.post("/v1/vault/links/batch", { doc_ids: docIds }), enabled: documents.length > 0, staleTime: 60_000, }); @@ -181,22 +145,34 @@ export function useVaultGraphData(agentId: string, opts?: { teamId?: string }) { return { documents, links: linksData ?? [], loading: docsLoading || linksLoading }; } +/** Enriched backlink with source doc metadata (from backend JOIN). */ +export interface VaultBacklink { + from_doc_id: string; + context: string; + title: string; + path: string; + team_id?: string; +} + /** Get links (outlinks + backlinks) for a vault document. */ -export function useVaultLinks(agentId: string, docId: string | null) { +export function useVaultLinks(docId: string | null) { const http = useHttp(); const { data, isLoading } = useQuery({ - queryKey: [VAULT_KEY, "links", agentId, docId], - queryFn: () => http.get<{ outlinks: VaultLink[]; backlinks: VaultLink[] }>( - `/v1/agents/${agentId}/vault/documents/${docId}/links`, - ), - enabled: !!docId && !!agentId, + queryKey: [VAULT_KEY, "links", docId], + queryFn: () => http.get<{ + outlinks: VaultLink[]; + backlinks: VaultBacklink[]; + doc_names: Record; + }>(`/v1/vault/documents/${docId}/links`), + enabled: !!docId, placeholderData: (prev) => prev, }); return { outlinks: data?.outlinks ?? [], backlinks: data?.backlinks ?? [], + docNames: data?.doc_names ?? {}, loading: isLoading, }; } @@ -219,6 +195,25 @@ export function useVaultFileContent(path: string | null) { return { content: data?.content ?? null, size: data?.size ?? 0, loading: isLoading, error: !!error }; } +/** Fetch an image file as blob URL for authenticated rendering in tags. */ +export function useVaultImageUrl(path: string | null): { url: string | null; error: boolean } { + const http = useHttp(); + const [url, setUrl] = useState(null); + const [error, setError] = useState(false); + + useEffect(() => { + if (!path) { setUrl(null); setError(false); return; } + let revoke: string | null = null; + setError(false); + http.downloadBlob(`/v1/storage/files/${encodeURIComponent(path)}?raw=true`) + .then((blob) => { revoke = URL.createObjectURL(blob); setUrl(revoke); }) + .catch(() => { setUrl(null); setError(true); }); + return () => { if (revoke) URL.revokeObjectURL(revoke); }; + }, [path, http]); + + return { url, error }; +} + // Re-export mutations for convenience — consumers can import from this single file export { useCreateDocument, @@ -226,4 +221,5 @@ export { useDeleteDocument, useCreateLink, useDeleteLink, + useRescanWorkspace, } from "./use-vault-mutations"; diff --git a/ui/web/src/pages/vault/vault-create-dialog.tsx b/ui/web/src/pages/vault/vault-create-dialog.tsx index 1daa852f..1c7657ad 100644 --- a/ui/web/src/pages/vault/vault-create-dialog.tsx +++ b/ui/web/src/pages/vault/vault-create-dialog.tsx @@ -1,142 +1,215 @@ -import { useState } from "react"; -import { useForm } from "react-hook-form"; -import { zodResolver } from "@hookform/resolvers/zod"; +import { useState, useEffect, useCallback, useRef, type ChangeEvent, type DragEvent } from "react"; import { useTranslation } from "react-i18next"; +import { Upload, X, FileText } from "lucide-react"; import { Button } from "@/components/ui/button"; -import { Input } from "@/components/ui/input"; import { Label } from "@/components/ui/label"; import { Dialog, DialogContent, DialogHeader, DialogTitle, DialogFooter, } from "@/components/ui/dialog"; -import { useCreateDocument } from "./hooks/use-vault"; -import { vaultCreateSchema, type VaultCreateFormData } from "@/schemas/vault.schema"; +import { useAgents } from "@/pages/agents/hooks/use-agents"; +import { useTeams } from "@/pages/teams/hooks/use-teams"; +import { useVaultUpload } from "./hooks/use-vault-upload"; -const DOC_TYPES = ["context", "memory", "note", "skill"] as const; -const SCOPES = ["personal", "team", "shared"] as const; +const ACCEPTED_EXTS = new Set([ + ".md", ".txt", ".json", ".yaml", ".yml", ".csv", ".toml", ".xml", ".html", ".htm", + ".go", ".py", ".js", ".ts", ".tsx", ".jsx", ".rs", ".java", ".rb", ".sh", ".sql", + ".swift", ".kt", ".c", ".cpp", ".h", +]); +const ACCEPT_ATTR = Array.from(ACCEPTED_EXTS).join(","); -interface Props { - agentId: string; - open: boolean; - onOpenChange: (open: boolean) => void; - onCreated?: () => void; +function isAllowedExt(name: string): boolean { + const dot = name.lastIndexOf("."); + if (dot < 0) return false; + return ACCEPTED_EXTS.has(name.slice(dot).toLowerCase()); } -export function VaultCreateDialog({ agentId, open, onOpenChange, onCreated }: Props) { +function formatBytes(bytes: number): string { + if (bytes < 1024) return `${bytes} B`; + if (bytes < 1024 * 1024) return `${(bytes / 1024).toFixed(1)} KB`; + return `${(bytes / (1024 * 1024)).toFixed(1)} MB`; +} + +type Destination = "shared" | "agent" | "team"; + +interface Props { + open: boolean; + onOpenChange: (open: boolean) => void; + onUploaded?: () => void; + defaultAgentId?: string; + defaultTeamId?: string; +} + +export function VaultCreateDialog({ open, onOpenChange, onUploaded, defaultAgentId, defaultTeamId }: Props) { const { t } = useTranslation("vault"); - const { create } = useCreateDocument(agentId); - const [saving, setSaving] = useState(false); + const { agents } = useAgents(); + const { teams, load: loadTeams } = useTeams(); + const { upload, isPending } = useVaultUpload(); + const inputRef = useRef(null); - const { - register, - handleSubmit, - watch, - setValue, - reset, - formState: { errors }, - } = useForm({ - resolver: zodResolver(vaultCreateSchema), - defaultValues: { - title: "", - path: "", - docType: "note", - scope: "personal", - }, - }); + const [destination, setDestination] = useState("shared"); + const [agentId, setAgentId] = useState(""); + const [teamId, setTeamId] = useState(""); + const [files, setFiles] = useState([]); + const [dragging, setDragging] = useState(false); - const docType = watch("docType"); - const scope = watch("scope"); + // Load teams on first open + useEffect(() => { if (open) loadTeams(); }, [open, loadTeams]); - const onValid = async (data: VaultCreateFormData) => { - setSaving(true); + // Reset state when dialog opens; pre-select destination from parent filters + useEffect(() => { + if (!open) return; + setFiles([]); + setDragging(false); + if (defaultAgentId) { + setDestination("agent"); + setAgentId(defaultAgentId); + } else if (defaultTeamId) { + setDestination("team"); + setTeamId(defaultTeamId); + } else { + setDestination("shared"); + } + }, [open, defaultAgentId, defaultTeamId]); + + const addFiles = useCallback((incoming: File[]) => { + const valid = incoming.filter((f) => isAllowedExt(f.name)); + setFiles((prev) => { + const names = new Set(prev.map((f) => f.name)); + return [...prev, ...valid.filter((f) => !names.has(f.name))]; + }); + }, []); + + const onDrop = useCallback((e: DragEvent) => { + e.preventDefault(); + setDragging(false); + addFiles(Array.from(e.dataTransfer.files)); + }, [addFiles]); + + const onFileInput = useCallback((e: ChangeEvent) => { + addFiles(Array.from(e.target.files ?? [])); + e.target.value = ""; + }, [addFiles]); + + const removeFile = (idx: number) => setFiles((prev) => prev.filter((_, i) => i !== idx)); + + const handleUpload = async () => { + const opts = destination === "agent" ? { agentId } + : destination === "team" ? { teamId } + : {}; try { - await create({ title: data.title.trim(), path: data.path.trim(), doc_type: data.docType, scope: data.scope }); - reset(); - onCreated?.(); + await upload(files, opts); onOpenChange(false); + onUploaded?.(); } catch { // error toasted in hook - } finally { - setSaving(false); } }; - const handleClose = (v: boolean) => { - if (!saving) { reset(); onOpenChange(v); } - }; + const canUpload = files.length > 0 + && !isPending + && (destination !== "agent" || agentId) + && (destination !== "team" || teamId); + + const handleClose = (v: boolean) => { if (!isPending) onOpenChange(v); }; return ( - {t("createDoc")} + {t("upload.title", "Upload to Vault")} - - - {t("fields.title")} * - - {errors.title && ( - {errors.title.message} - )} - + + {/* Destination radio group */} + + {t("upload.destination", "Destination")} + + + setDestination("shared")} className="accent-primary" /> + {t("upload.shared", "Shared")} + - - {t("fields.path")} * - - {errors.path && ( - {errors.path.message} - )} - + + setDestination("agent")} className="accent-primary" /> + {t("upload.agent", "Agent")} + + {destination === "agent" && ( + setAgentId(e.target.value)} + className="ml-6 w-[calc(100%-1.5rem)] text-base md:text-sm border rounded px-2 py-1.5 bg-background"> + {t("upload.selectAgent", "Select agent...")} + {(agents ?? []).map((a) => ( + {a.display_name || a.agent_key} + ))} + + )} - - - {t("fields.docType")} - setValue("docType", e.target.value, { shouldValidate: true })} - className="w-full text-base md:text-sm border rounded px-2 py-1.5 bg-background" - > - {DOC_TYPES.map((dt) => ( - {t(`type.${dt}`)} - ))} - + + setDestination("team")} className="accent-primary" /> + {t("upload.team", "Team")} + + {destination === "team" && ( + setTeamId(e.target.value)} + className="ml-6 w-[calc(100%-1.5rem)] text-base md:text-sm border rounded px-2 py-1.5 bg-background"> + {t("upload.selectTeam", "Select team...")} + {(teams ?? []).map((team) => ( + {team.name} + ))} + + )} + - - {t("fields.scope")} - setValue("scope", e.target.value, { shouldValidate: true })} - className="w-full text-base md:text-sm border rounded px-2 py-1.5 bg-background" - > - {SCOPES.map((s) => ( - {t(`scope.${s}`)} - ))} - - + {/* Drop zone */} + { e.preventDefault(); setDragging(true); }} + onDragLeave={() => setDragging(false)} + onDrop={onDrop} + onClick={() => inputRef.current?.click()} + className={`flex flex-col items-center justify-center gap-2 rounded-lg border-2 border-dashed p-6 cursor-pointer transition-colors ${ + dragging ? "border-primary bg-primary/5" : "border-muted-foreground/25 hover:border-primary/50" + }`} + > + + {t("upload.dropzone", "Drop files here or click to browse")} + {t("upload.dropzoneHint", "Text files only (.md, .txt, .json, .yaml, .csv, ...)")} + - - { reset(); onOpenChange(false); }} disabled={saving}> - {t("cancel")} - - - {saving ? t("saving") : t("create")} - - - + {/* File list */} + {files.length > 0 && ( + + + {t("upload.files", { count: files.length, defaultValue: `Files (${files.length})` })} + + + {files.map((f, i) => ( + + + {f.name} + {formatBytes(f.size)} + removeFile(i)} + className="shrink-0 p-0.5 rounded hover:bg-destructive/10 text-muted-foreground hover:text-destructive"> + + + + ))} + + + )} + + + + onOpenChange(false)} disabled={isPending}> + {t("cancel")} + + + {isPending ? t("upload.uploading", "Uploading...") : t("upload.upload", "Upload")} + + ); diff --git a/ui/web/src/pages/vault/vault-detail-dialog.tsx b/ui/web/src/pages/vault/vault-detail-dialog.tsx index 102a0092..a5adfbcf 100644 --- a/ui/web/src/pages/vault/vault-detail-dialog.tsx +++ b/ui/web/src/pages/vault/vault-detail-dialog.tsx @@ -1,6 +1,6 @@ -import { useState } from "react"; +import { useState, useMemo } from "react"; import { useTranslation } from "react-i18next"; -import { Pencil, Plus, FileText, Link2 } from "lucide-react"; +import { Pencil, Plus, FileText, Link2, FileQuestion } from "lucide-react"; import { Badge } from "@/components/ui/badge"; import { Button } from "@/components/ui/button"; import { Input } from "@/components/ui/input"; @@ -8,7 +8,7 @@ import { Dialog, DialogContent, DialogHeader, DialogTitle, } from "@/components/ui/dialog"; import { MarkdownRenderer } from "@/components/shared/markdown-renderer"; -import { useVaultLinks, useVaultFileContent, useUpdateDocument, useDeleteDocument } from "./hooks/use-vault"; +import { useVaultLinks, useVaultFileContent, useVaultImageUrl, useUpdateDocument, useDeleteDocument } from "./hooks/use-vault"; import { VaultLinkDialog } from "./vault-link-dialog"; import { VaultEditControls, DocTypeSelect, ScopeSelect, LinkBadge, @@ -24,7 +24,7 @@ interface Props { export function VaultDetailDialog({ doc, open, onOpenChange, onDeleted }: Props) { const { t } = useTranslation("vault"); - const { outlinks, backlinks, loading } = useVaultLinks(doc?.agent_id ?? "", doc?.id ?? null); + const { outlinks, backlinks, docNames, loading } = useVaultLinks(doc?.id ?? null); const [editMode, setEditMode] = useState(false); const [editTitle, setEditTitle] = useState(""); @@ -34,12 +34,24 @@ export function VaultDetailDialog({ doc, open, onOpenChange, onDeleted }: Props) const [confirmDelete, setConfirmDelete] = useState(false); const [linkDialogOpen, setLinkDialogOpen] = useState(false); - // Content always loaded when dialog is open + // Determine if this is an image that can be rendered. + const isImage = useMemo(() => { + const mime = doc?.metadata?.mime_type as string | undefined; + if (mime?.startsWith("image/")) return true; + const ext = doc?.path.split(".").pop()?.toLowerCase() ?? ""; + return ["png", "jpg", "jpeg", "gif", "webp", "svg", "bmp", "ico"].includes(ext); + }, [doc?.metadata, doc?.path]); + + const isMedia = doc?.doc_type === "media"; + + // Only fetch text content for non-media files (media/binary cannot be rendered as markdown). const { content: fileContent, loading: contentLoading, error: contentError } = useVaultFileContent( - open && doc ? doc.path : null, + open && doc && !isMedia ? doc.path : null, ); - const { update } = useUpdateDocument(doc?.agent_id ?? "", doc?.id ?? ""); - const { remove: removeDoc } = useDeleteDocument(doc?.agent_id ?? "", doc?.id ?? ""); + // Fetch image as authenticated blob URL for rendering. + const { url: imageUrl, error: imageError } = useVaultImageUrl(open && isMedia && isImage && doc ? doc.path : null); + const { update } = useUpdateDocument(doc?.id ?? ""); + const { remove: removeDoc } = useDeleteDocument(doc?.id ?? ""); if (!doc) return null; @@ -149,7 +161,37 @@ export function VaultDetailDialog({ doc, open, onOpenChange, onDeleted }: Props) {/* Content preview — always visible, scrollable */} - {contentLoading ? ( + {isMedia ? ( + isImage ? ( + + {imageUrl ? ( + + ) : imageError ? ( + + + {t("detail.fileNotFound")} + + ) : ( + + )} + + ) : ( + + + + {t("detail.binaryFile")} + {(doc.metadata?.mime_type as string) || doc.path.split(".").pop()?.toUpperCase()} + + {doc.summary && ( + {doc.summary} + )} + + ) + ) : contentLoading ? ( @@ -185,38 +227,40 @@ export function VaultDetailDialog({ doc, open, onOpenChange, onDeleted }: Props) {t("detail.outlinks")} ({outlinks.length}) {outlinks.length > 0 ? ( - + {outlinks.map((l) => ( - + ))} ) : ( {t("detail.noLinks")} )} - setLinkDialogOpen(true)}> - - {t("addLink")} - )} {!loading && backlinks.length > 0 && ( - + {t("detail.backlinks")} ({backlinks.length}) - + {backlinks.map((l) => ( - - {l.link_type}: {l.from_doc_id.slice(0, 8)} + + {l.title || l.from_doc_id.slice(0, 8)} ))} )} - {/* Hash + link count as subtle footer */} + {/* Bottom bar: hash + add link */} - SHA-256: {doc.content_hash.length > 16 ? `${doc.content_hash.slice(0, 8)}...${doc.content_hash.slice(-8)}` : doc.content_hash} - {linkCount > 0 && {linkCount} link{linkCount !== 1 ? "s" : ""}} + + SHA-256: {doc.content_hash.length > 16 ? `${doc.content_hash.slice(0, 8)}...${doc.content_hash.slice(-8)}` : doc.content_hash} + {linkCount > 0 && {linkCount} link{linkCount !== 1 ? "s" : ""}} + + setLinkDialogOpen(true)}> + + {t("addLink")} + {/* Metadata JSON (if present) */} @@ -236,7 +280,7 @@ export function VaultDetailDialog({ doc, open, onOpenChange, onDeleted }: Props) {linkDialogOpen && ( string }) { +export function LinkBadge({ link, docNames, t }: { link: VaultLink; docNames?: Record; t: (k: string) => string }) { const [confirmDelete, setConfirmDelete] = useState(false); - const { remove } = useDeleteLink(agentId, link.id); + const { remove } = useDeleteLink(link.id); const handleDelete = async () => { try { @@ -123,8 +123,8 @@ export function LinkBadge({ link, agentId, t }: { link: VaultLink; agentId: stri } return ( - - {link.link_type}: {link.to_doc_id.slice(0, 8)} + + {link.link_type}: {docNames?.[link.to_doc_id] || link.to_doc_id.slice(0, 8)} setConfirmDelete(true)} className="opacity-0 group-hover:opacity-100 transition-opacity" diff --git a/ui/web/src/pages/vault/vault-detail-panel.tsx b/ui/web/src/pages/vault/vault-detail-panel.tsx index 54c4567e..300ca341 100644 --- a/ui/web/src/pages/vault/vault-detail-panel.tsx +++ b/ui/web/src/pages/vault/vault-detail-panel.tsx @@ -19,10 +19,10 @@ interface Props { export function VaultDetailPanel({ doc, open, onClose, onDeleted }: Props) { const { t } = useTranslation("vault"); - const { outlinks, backlinks } = useVaultLinks(doc?.agent_id ?? "", doc?.id ?? null); + const { outlinks, backlinks, docNames } = useVaultLinks(doc?.id ?? null); const { content, loading: contentLoading, error: contentError } = useVaultFileContent(open && doc ? doc.path : null); - const { update } = useUpdateDocument(doc?.agent_id ?? "", doc?.id ?? ""); - const { remove } = useDeleteDocument(doc?.agent_id ?? "", doc?.id ?? ""); + const { update } = useUpdateDocument(doc?.id ?? ""); + const { remove } = useDeleteDocument(doc?.id ?? ""); const [editMode, setEditMode] = useState(false); const [editTitle, setEditTitle] = useState(""); @@ -151,7 +151,7 @@ export function VaultDetailPanel({ doc, open, onClose, onDeleted }: Props) { {outlinks.length > 0 && ( - {outlinks.map((l) => )} + {outlinks.map((l) => )} )} @@ -162,7 +162,9 @@ export function VaultDetailPanel({ doc, open, onClose, onDeleted }: Props) { {t("detail.backlinks")} ({backlinks.length}) {backlinks.map((l) => ( - {l.from_doc_id.slice(0, 8)} + + {l.title || l.from_doc_id.slice(0, 8)} + ))} @@ -172,7 +174,7 @@ export function VaultDetailPanel({ doc, open, onClose, onDeleted }: Props) { {linkDialogOpen && ( - )} > diff --git a/ui/web/src/pages/vault/vault-document-sidebar.tsx b/ui/web/src/pages/vault/vault-document-sidebar.tsx index 380864b1..ae1a154e 100644 --- a/ui/web/src/pages/vault/vault-document-sidebar.tsx +++ b/ui/web/src/pages/vault/vault-document-sidebar.tsx @@ -1,24 +1,26 @@ -import { useEffect, useRef } from "react"; +import { useState, useEffect, useRef, useCallback } from "react"; import { useTranslation } from "react-i18next"; import { - Link2, ChevronLeft, ChevronRight, + Link2, ChevronLeft, ChevronRight, Search, X, Loader2, FileText, Brain, StickyNote, Sparkles, Clock, Image, } from "lucide-react"; import { Badge } from "@/components/ui/badge"; import { Button } from "@/components/ui/button"; import { formatRelativeTime } from "@/lib/format"; -import type { VaultDocument } from "@/types/vault"; +import type { VaultDocument, VaultSearchResult } from "@/types/vault"; +import { useVaultSearchAll } from "./hooks/use-vault"; -const DOC_TYPE_CONFIG: Record = { - context: { color: "text-blue-600 dark:text-blue-400", bg: "bg-blue-500/10", icon: FileText }, - memory: { color: "text-purple-600 dark:text-purple-400", bg: "bg-purple-500/10", icon: Brain }, - note: { color: "text-amber-600 dark:text-amber-400", bg: "bg-amber-500/10", icon: StickyNote }, - skill: { color: "text-emerald-600 dark:text-emerald-400", bg: "bg-emerald-500/10", icon: Sparkles }, - episodic: { color: "text-orange-600 dark:text-orange-400", bg: "bg-orange-500/10", icon: Clock }, - media: { color: "text-rose-600 dark:text-rose-400", bg: "bg-rose-500/10", icon: Image }, +const DOC_TYPE_CONFIG: Record = { + context: { color: "text-blue-600 dark:text-blue-400", bg: "bg-blue-500/10", icon: FileText, dotColor: "bg-blue-500" }, + memory: { color: "text-purple-600 dark:text-purple-400", bg: "bg-purple-500/10", icon: Brain, dotColor: "bg-purple-500" }, + note: { color: "text-amber-600 dark:text-amber-400", bg: "bg-amber-500/10", icon: StickyNote, dotColor: "bg-amber-500" }, + skill: { color: "text-emerald-600 dark:text-emerald-400", bg: "bg-emerald-500/10", icon: Sparkles, dotColor: "bg-emerald-500" }, + episodic: { color: "text-orange-600 dark:text-orange-400", bg: "bg-orange-500/10", icon: Clock, dotColor: "bg-orange-500" }, + media: { color: "text-rose-600 dark:text-rose-400", bg: "bg-rose-500/10", icon: Image, dotColor: "bg-rose-500" }, }; -const DEFAULT_CONFIG = { color: "text-muted-foreground", bg: "bg-muted", icon: FileText }; +const DEFAULT_CONFIG = { color: "text-muted-foreground", bg: "bg-muted", icon: FileText, dotColor: "bg-muted-foreground" }; +const DOC_TYPES = ["context", "memory", "note", "skill", "episodic", "media"] as const; interface Props { documents: VaultDocument[]; @@ -30,6 +32,10 @@ interface Props { totalPages: number; total: number; onPageChange: (page: number) => void; + docType: string; + onDocTypeChange: (type: string) => void; + agentId: string; + teamId: string; } function DocCard({ doc, selected, linkCount, onClick }: { @@ -54,17 +60,13 @@ function DocCard({ doc, selected, linkCount, onClick }: { }`} onClick={onClick} > - {/* Type icon */} - - {/* Content */} {doc.title || doc.path.split("/").pop()} - {t(`type.${doc.doc_type}`)} · @@ -87,21 +89,125 @@ function DocCard({ doc, selected, linkCount, onClick }: { } export function VaultDocumentSidebar({ - documents, selectedId, linkCounts, onSelect, loading, page, totalPages, total, onPageChange, + documents, selectedId, linkCounts, onSelect, loading, + page, totalPages, total, onPageChange, + docType, onDocTypeChange, agentId, teamId, }: Props) { const { t } = useTranslation("vault"); + const [query, setQuery] = useState(""); + const [searchResults, setSearchResults] = useState(null); + const [searching, setSearching] = useState(false); + const inputRef = useRef(null); + const debounceRef = useRef>(null); + const { search } = useVaultSearchAll(); + + const doSearch = useCallback( + async (q: string) => { + if (!q.trim()) { + setSearchResults(null); + return; + } + setSearching(true); + try { + const results = await search(q, { + agentId: agentId || undefined, + docTypes: docType ? [docType] : undefined, + teamId: teamId || undefined, + maxResults: 30, + }); + setSearchResults(results); + } catch { + setSearchResults([]); + } finally { + setSearching(false); + } + }, + [search, agentId, docType, teamId], + ); + + const handleQueryChange = (value: string) => { + setQuery(value); + if (debounceRef.current) clearTimeout(debounceRef.current); + if (!value.trim()) { + setSearchResults(null); + return; + } + debounceRef.current = setTimeout(() => doSearch(value), 300); + }; + + const clearSearch = () => { + setQuery(""); + setSearchResults(null); + inputRef.current?.focus(); + }; + + // Show search results or regular document list + const isSearchMode = query.trim().length > 0; + const displayDocs = isSearchMode && searchResults + ? searchResults.map((r) => r.document) + : documents; return ( - {/* Header */} - - {t("title")} - {total} + {/* Filter bar: type chips + search */} + + {/* Type filter chips — horizontal scroll */} + + onDocTypeChange("")} + className={`shrink-0 px-1.5 py-0.5 rounded text-2xs font-medium transition-colors ${ + !docType ? "bg-primary text-primary-foreground" : "bg-muted hover:bg-muted/80 text-muted-foreground" + }`} + > + {t("allTypes")} + + {DOC_TYPES.map((dt) => { + const cfg = DOC_TYPE_CONFIG[dt] ?? DEFAULT_CONFIG; + const active = docType === dt; + return ( + onDocTypeChange(active ? "" : dt)} + className={`shrink-0 flex items-center gap-1 px-1.5 py-0.5 rounded text-2xs font-medium transition-colors ${ + active ? "bg-primary text-primary-foreground" : "bg-muted hover:bg-muted/80 text-muted-foreground" + }`} + > + + {t(`type.${dt}`)} + + ); + })} + {total} + + + {/* Search input */} + + {searching + ? + : + } + handleQueryChange(e.target.value)} + placeholder={t("searchPlaceholder")} + className="w-full rounded-md border bg-background pl-7 pr-7 py-1 text-base md:text-xs placeholder:text-muted-foreground focus:outline-none focus:ring-1 focus:ring-ring" + /> + {query && ( + + + + )} + {/* Doc list */} - {loading && documents.length === 0 ? ( + {(loading || searching) && displayDocs.length === 0 ? ( Array.from({ length: 6 }).map((_, i) => ( @@ -111,13 +217,13 @@ export function VaultDocumentSidebar({ )) - ) : documents.length === 0 ? ( + ) : displayDocs.length === 0 ? ( - {t("noDocuments")} + {isSearchMode ? t("noResults") : t("noDocuments")} ) : ( - documents.map((doc) => ( + displayDocs.map((doc) => ( - {/* Pagination footer */} - {totalPages > 1 && ( + {/* Pagination footer — hidden during search */} + {!isSearchMode && totalPages > 1 && ( onPageChange(page - 1)}> diff --git a/ui/web/src/pages/vault/vault-documents-table.tsx b/ui/web/src/pages/vault/vault-documents-table.tsx index 385d0f6c..6a949982 100644 --- a/ui/web/src/pages/vault/vault-documents-table.tsx +++ b/ui/web/src/pages/vault/vault-documents-table.tsx @@ -80,7 +80,7 @@ export function VaultDocumentsTable({ documents, agents, loading, onSelect }: Pr {doc.title || doc.path.split("/").pop()} - {agentMap.get(doc.agent_id) ?? doc.agent_id.slice(0, 8)} + {doc.agent_id ? (agentMap.get(doc.agent_id) ?? doc.agent_id.slice(0, 8)) : t("scope.shared")} {truncatePath(doc.path)} diff --git a/ui/web/src/pages/vault/vault-graph-controls.tsx b/ui/web/src/pages/vault/vault-graph-controls.tsx deleted file mode 100644 index cd7994db..00000000 --- a/ui/web/src/pages/vault/vault-graph-controls.tsx +++ /dev/null @@ -1,57 +0,0 @@ -import type { RefObject } from "react"; -import { ZoomIn, ZoomOut, Maximize2 } from "lucide-react"; -import { Button } from "@/components/ui/button"; -import { useTranslation } from "react-i18next"; - -const NODE_LIMIT_OPTIONS = [100, 200, 300, 500] as const; - -interface Props { - docCount: number; - linkCount: number; - nodeLimit: number; - isLimited: boolean; - zoomDisplayRef: RefObject; - onNodeLimitChange: (limit: number) => void; - onZoomIn: () => void; - onZoomOut: () => void; - onFitToView: () => void; -} - -export function VaultGraphControls({ - docCount, linkCount, nodeLimit, isLimited, zoomDisplayRef, - onNodeLimitChange, onZoomIn, onZoomOut, onFitToView, -}: Props) { - const { t } = useTranslation("vault"); - - return ( - - {t("graphDocs", { count: docCount, defaultValue: "{{count}} docs" })} - {t("graphLinks", { count: linkCount, defaultValue: "{{count}} links" })} - {isLimited && ( - · {t("graphLimitNote", { limit: nodeLimit, total: docCount, defaultValue: "showing {{limit}} of {{total}}" })} - )} - - - - - - 100% - - - - - onNodeLimitChange(Number(e.target.value))} - className="h-5 rounded border bg-background px-1 text-base md:text-2xs" - > - {NODE_LIMIT_OPTIONS.map((n) => ( - {n} nodes - ))} - - - - - - ); -} diff --git a/ui/web/src/pages/vault/vault-graph-view.tsx b/ui/web/src/pages/vault/vault-graph-view.tsx index 261f82df..922c904a 100644 --- a/ui/web/src/pages/vault/vault-graph-view.tsx +++ b/ui/web/src/pages/vault/vault-graph-view.tsx @@ -1,14 +1,17 @@ -import { useRef, useMemo, useEffect, useLayoutEffect, useCallback, useState } from "react"; -import ForceGraph2D, { type ForceGraphMethods } from "react-force-graph-2d"; -import { useUiStore } from "@/stores/use-ui-store"; -import { buildVaultGraphData, limitVaultDocsByDegree, VAULT_TYPE_COLORS } from "@/adapters/vault-graph-adapter"; -import type { VaultGraphNode, VaultGraphLink } from "@/adapters/vault-graph-adapter"; +import { useRef, useMemo, useState, useCallback } from "react"; +import type Sigma from "sigma"; +import { useTranslation } from "react-i18next"; +import { buildVaultGraph, limitVaultDocsByDegree, VAULT_TYPE_COLORS } from "@/adapters/vault-graph-adapter"; +import { SigmaGraphContainer } from "@/components/graph/sigma-graph-container"; +import { SigmaGraphControls } from "@/components/graph/sigma-graph-controls"; +import { SigmaGraphSearch } from "@/components/graph/sigma-graph-search"; +import { SigmaGraphFilters } from "@/components/graph/sigma-graph-filters"; +import { SigmaGraphMinimap } from "@/components/graph/sigma-graph-minimap"; +import { SigmaGraphKeyboardHelp } from "@/components/graph/sigma-graph-keyboard-help"; +import { useSigmaKeyboard } from "@/components/graph/use-sigma-keyboard"; import type { VaultDocument } from "@/types/vault"; import { useVaultGraphData } from "./hooks/use-vault"; -import { VaultGraphControls } from "./vault-graph-controls"; -const NODE_R = 5; -const DOUBLE_CLICK_MS = 280; const DEFAULT_NODE_LIMIT = 200; interface Props { @@ -20,192 +23,119 @@ interface Props { } export function VaultGraphView({ agentId, teamId, selectedDocId, onNodeSelect, onNodeDoubleClick }: Props) { - const theme = useUiStore((s) => s.theme); - const isDark = theme === "dark" || (theme === "system" && window.matchMedia("(prefers-color-scheme: dark)").matches); + const { t } = useTranslation("vault"); + const containerRef = useRef(null); + const searchInputRef = useRef(null); + const [sigma, setSigma] = useState(null); + const [nodeLimit, setNodeLimit] = useState(DEFAULT_NODE_LIMIT); + const [hiddenTypes, setHiddenTypes] = useState>(new Set()); + const [filtersOpen, setFiltersOpen] = useState(false); const { documents: allDocs, links, loading } = useVaultGraphData(agentId, { teamId }); - const graphRef = useRef>(undefined); - const containerRef = useRef(null); - const [dimensions, setDimensions] = useState({ width: 600, height: 400 }); - const [ready, setReady] = useState(false); - const [nodeLimit, setNodeLimit] = useState(DEFAULT_NODE_LIMIT); - const zoomRef = useRef(1); - const zoomDisplayRef = useRef(null); - const clickTimerRef = useRef | null>(null); - const lastClickRef = useRef(null); - - // Container sizing - useLayoutEffect(() => { - const el = containerRef.current; - if (el && el.clientWidth > 0 && el.clientHeight > 0) { - setDimensions({ width: el.clientWidth, height: el.clientHeight }); - setReady(true); - } - }, []); - - useEffect(() => { - const el = containerRef.current; - if (!el) return; - const ro = new ResizeObserver(([entry]) => { - if (!entry) return; - const { width, height } = entry.contentRect; - if (width > 0 && height > 0) { setDimensions({ width, height }); setReady(true); } - }); - ro.observe(el); - return () => ro.disconnect(); - }, []); - - // Pan camera to externally-selected node - useEffect(() => { - if (!selectedDocId || !graphRef.current) return; - const node = graphData.nodes.find((n) => n.id === selectedDocId); - if (node?.x != null && node?.y != null) { - graphRef.current.centerAt(node.x, node.y, 300); - } - }, [selectedDocId]); - const totalCount = allDocs.length; const isLimited = totalCount > nodeLimit; const documents = useMemo(() => limitVaultDocsByDegree(allDocs, links, nodeLimit), [allDocs, links, nodeLimit]); const docMap = useMemo(() => new Map(documents.map((d) => [d.id, d])), [documents]); - const graphData = useMemo(() => buildVaultGraphData(documents, links), [documents, links]); - - // Highlight sets from selectedDocId (controlled by parent) - const { highlightNodes, highlightLinks } = useMemo(() => { - if (!selectedDocId) return { highlightNodes: new Set(), highlightLinks: new Set() }; - const node = graphData.nodes.find((n) => n.id === selectedDocId); - if (!node) return { highlightNodes: new Set(), highlightLinks: new Set() }; - return { highlightNodes: new Set([selectedDocId, ...node.neighbors]), highlightLinks: new Set(node.linkIds) }; - }, [selectedDocId, graphData.nodes]); - - const nodeCanvasObject = useCallback( - (node: VaultGraphNode, ctx: CanvasRenderingContext2D, globalScale: number) => { - const x = node.x ?? 0, y = node.y ?? 0; - const isSelected = node.id === selectedDocId; - const isNeighbor = highlightNodes.has(node.id); - const isDimmed = !!selectedDocId && !isSelected && !isNeighbor; - const r = NODE_R + Math.min(node.degree, 10); - - ctx.beginPath(); - ctx.arc(x, y, r, 0, 2 * Math.PI); - ctx.fillStyle = isDimmed ? `${node.color}30` : node.color; - ctx.fill(); - - if (isSelected) { - ctx.strokeStyle = node.color; ctx.lineWidth = 2.5 / globalScale; - ctx.shadowColor = node.color; ctx.shadowBlur = 8 / globalScale; - ctx.stroke(); ctx.shadowBlur = 0; - } else if (isNeighbor) { - ctx.strokeStyle = node.color; ctx.lineWidth = 1.5 / globalScale; ctx.stroke(); - } - - if (globalScale > 0.5 || isSelected || isNeighbor) { - const fontSize = Math.max(11 / globalScale, 2); - ctx.font = `${isSelected ? "bold " : ""}${fontSize}px Inter, system-ui, sans-serif`; - ctx.textAlign = "center"; ctx.textBaseline = "top"; - ctx.fillStyle = isDimmed - ? (isDark ? "rgba(255,255,255,0.12)" : "rgba(0,0,0,0.12)") - : (isDark ? "#e2e8f0" : "#1e293b"); - ctx.fillText(node.title.slice(0, 24), x, y + r + 2 / globalScale); - } - }, - [selectedDocId, highlightNodes, isDark], + // Only build graph when ALL data loaded — prevents double-render + // (orphan-only layout → with-links layout) that causes visual chaos. + const graph = useMemo( + () => loading ? buildVaultGraph([], []) : buildVaultGraph(documents, links), + [documents, links, loading], ); - const linkCanvasObject = useCallback( - (link: any, ctx: CanvasRenderingContext2D, globalScale: number) => { - if (!highlightLinks.has(link.id)) return; - const src = link.source, tgt = link.target; - if (!src?.x || !tgt?.x) return; - const midX = (src.x + tgt.x) / 2, midY = (src.y + tgt.y) / 2; - const fontSize = Math.max(9 / globalScale, 2); - const text = link.label || ""; - ctx.font = `${fontSize}px Inter, system-ui, sans-serif`; - ctx.textAlign = "center"; ctx.textBaseline = "middle"; - const metrics = ctx.measureText(text); - const pad = 2 / globalScale; - ctx.fillStyle = isDark ? "rgba(15,23,42,0.85)" : "rgba(255,255,255,0.9)"; - ctx.fillRect(midX - metrics.width / 2 - pad, midY - fontSize / 2 - pad, metrics.width + pad * 2, fontSize + pad * 2); - ctx.fillStyle = isDark ? "#94a3b8" : "#64748b"; - ctx.fillText(text, midX, midY); - }, - [highlightLinks, isDark], - ); + const handleNodeDoubleClick = useCallback((nodeId: string) => { + const doc = docMap.get(nodeId); + if (doc) onNodeDoubleClick?.(doc); + }, [docMap, onNodeDoubleClick]); - // Single-click = highlight (notify parent), double-click = open detail - const handleNodeClick = useCallback( - (node: VaultGraphNode) => { - if (clickTimerRef.current && lastClickRef.current === node.id) { - clearTimeout(clickTimerRef.current); clickTimerRef.current = null; lastClickRef.current = null; - const doc = docMap.get(node.id); - if (doc) onNodeDoubleClick?.(doc); - return; - } - if (clickTimerRef.current) clearTimeout(clickTimerRef.current); - lastClickRef.current = node.id; - clickTimerRef.current = setTimeout(() => { - onNodeSelect?.(node.id === selectedDocId ? null : node.id); - clickTimerRef.current = null; - }, DOUBLE_CLICK_MS); - }, - [docMap, selectedDocId, onNodeSelect, onNodeDoubleClick], - ); + useSigmaKeyboard({ + sigma, + graph, + containerRef, + selectedNodeId: selectedDocId, + onNodeSelect, + searchInputRef, + }); - const handleZoomIn = useCallback(() => { const fg = graphRef.current; if (fg) fg.zoom(Math.min((fg.zoom() ?? 1) * 1.5, 8), 300); }, []); - const handleZoomOut = useCallback(() => { const fg = graphRef.current; if (fg) fg.zoom(Math.max((fg.zoom() ?? 1) / 1.5, 0.1), 300); }, []); - const handleFitToView = useCallback(() => graphRef.current?.zoomToFit(300, 20), []); + const hasData = allDocs.length > 0; return ( - - {/* Legend */} - - {Object.entries(VAULT_TYPE_COLORS).map(([type, color]) => ( - - - {type} - - ))} + + {/* Top bar — responsive: stacks on narrow screens */} + + + {Object.entries(VAULT_TYPE_COLORS).map(([type, color]) => ( + + + {type} + + ))} + + {hasData && ( + + + setFiltersOpen(!c)} + /> + + + )} - {/* Graph canvas */} - + {/* Graph canvas + minimap overlay */} + {loading && allDocs.length === 0 ? ( - ) : allDocs.length === 0 ? ( + ) : !hasData ? ( No documents - ) : ready ? ( - "replace"} - nodePointerAreaPaint={(node, color, ctx) => { - const r = NODE_R + Math.min((node as VaultGraphNode).degree, 10) + 3; - ctx.beginPath(); ctx.arc(node.x ?? 0, node.y ?? 0, r, 0, 2 * Math.PI); ctx.fillStyle = color; ctx.fill(); - }} - onNodeClick={handleNodeClick} - onBackgroundClick={() => onNodeSelect?.(null)} - onZoom={(transform) => { - zoomRef.current = transform.k; - if (zoomDisplayRef.current) zoomDisplayRef.current.textContent = `${Math.round(transform.k * 100)}%`; - }} - linkColor={(link: any) => highlightLinks.has(link.id) ? "#64748b" : (isDark ? "#334155" : "#cbd5e1")} - linkWidth={(link: any) => (highlightLinks.has(link.id) ? 2 : 0.5)} - linkCanvasObject={linkCanvasObject} - linkCanvasObjectMode={(link: any) => (highlightLinks.has(link.id) ? "after" : undefined)} - linkDirectionalParticles={(link: any) => (highlightLinks.has(link.id) ? 2 : 0)} - linkDirectionalParticleWidth={3} linkDirectionalParticleSpeed={0.004} - linkDirectionalArrowLength={4} linkDirectionalArrowRelPos={1} - backgroundColor="transparent" d3AlphaDecay={0.04} d3VelocityDecay={0.3} - warmupTicks={40} cooldownTime={4000} enableNodeDrag minZoom={0.1} maxZoom={8} - /> - ) : null} + ) : ( + <> + + + + + > + )} {/* Stats bar */} - ); diff --git a/ui/web/src/pages/vault/vault-link-dialog.tsx b/ui/web/src/pages/vault/vault-link-dialog.tsx index 277ac4d0..4c71aeeb 100644 --- a/ui/web/src/pages/vault/vault-link-dialog.tsx +++ b/ui/web/src/pages/vault/vault-link-dialog.tsx @@ -31,7 +31,7 @@ interface Props { export function VaultLinkDialog({ agentId, fromDoc, open, onOpenChange, onCreated }: Props) { const { t } = useTranslation("vault"); - const { create } = useCreateLink(agentId); + const { create } = useCreateLink(); const { documents } = useVaultDocuments(agentId, { limit: 100 }); const [saving, setSaving] = useState(false); diff --git a/ui/web/src/pages/vault/vault-page.tsx b/ui/web/src/pages/vault/vault-page.tsx index d0e8a3ca..057a67aa 100644 --- a/ui/web/src/pages/vault/vault-page.tsx +++ b/ui/web/src/pages/vault/vault-page.tsx @@ -1,12 +1,13 @@ -import { useState, useEffect, useMemo, lazy, Suspense } from "react"; +import { useState, useEffect, useMemo, useCallback, lazy, Suspense } from "react"; import { useTranslation } from "react-i18next"; -import { Search, FileArchive, Plus, PanelLeftOpen } from "lucide-react"; +import { Search, FileArchive, Plus, PanelLeftOpen, FolderSync, Loader2 } from "lucide-react"; import { Button } from "@/components/ui/button"; import { Tooltip, TooltipContent, TooltipProvider, TooltipTrigger } from "@/components/ui/tooltip"; import { useAgents } from "@/pages/agents/hooks/use-agents"; import { useTeams } from "@/pages/teams/hooks/use-teams"; import { useIsMobile } from "@/hooks/use-media-query"; -import { useVaultDocuments, useVaultGraphData } from "./hooks/use-vault"; +import { useVaultDocuments, useVaultGraphData, useRescanWorkspace } from "./hooks/use-vault"; +import { useEnrichmentProgress } from "./hooks/use-enrichment-progress"; import { VaultDocumentSidebar } from "./vault-document-sidebar"; import { VaultSearchDialog } from "./vault-search-dialog"; import { VaultCreateDialog } from "./vault-create-dialog"; @@ -31,6 +32,7 @@ export function VaultPage() { const [selectedAgent, setSelectedAgent] = useState(""); const [selectedTeam, setSelectedTeam] = useState(""); + const [docType, setDocType] = useState(""); const [detailDoc, setDetailDoc] = useState(null); const [selectedDocId, setSelectedDocId] = useState(null); const [searchOpen, setSearchOpen] = useState(false); @@ -38,8 +40,13 @@ export function VaultPage() { const [sidebarOpen, setSidebarOpen] = useState(false); const [page, setPage] = useState(0); + const { rescan, isPending: rescanPending } = useRescanWorkspace(); + const enrichment = useEnrichmentProgress(); + const enriching = enrichment?.running ?? false; + const { documents, total, loading } = useVaultDocuments(selectedAgent, { teamId: selectedTeam || undefined, + docType: docType || undefined, limit: PAGE_SIZE, offset: page * PAGE_SIZE, }); @@ -58,6 +65,7 @@ export function VaultPage() { const handleAgentChange = (v: string) => { setSelectedAgent(v); setPage(0); }; const handleTeamChange = (v: string) => { setSelectedTeam(v); setPage(0); }; + const handleDocTypeChange = (v: string) => { setDocType(v); setPage(0); }; // Sidebar click → open detail modal + highlight graph node const handleSidebarSelect = (doc: VaultDocument) => { @@ -67,15 +75,15 @@ export function VaultPage() { }; // Graph single-click → highlight only - const handleNodeSelect = (docId: string | null) => { + const handleNodeSelect = useCallback((docId: string | null) => { setSelectedDocId(docId); - }; + }, []); // Graph double-click → open detail modal + highlight - const handleNodeDoubleClick = (doc: VaultDocument) => { + const handleNodeDoubleClick = useCallback((doc: VaultDocument) => { setDetailDoc(doc); setSelectedDocId(doc.id); - }; + }, []); const handleCloseDetail = () => { setDetailDoc(null); }; @@ -102,6 +110,10 @@ export function VaultPage() { totalPages={totalPages} total={total} onPageChange={setPage} + docType={docType} + onDocTypeChange={handleDocTypeChange} + agentId={selectedAgent} + teamId={selectedTeam} /> @@ -131,20 +143,46 @@ export function VaultPage() { setSearchOpen(true)} disabled={!selectedAgent}> + + + + rescan()} disabled={rescanPending || enriching}> + {(rescanPending || enriching) ? : } + + + {enriching ? t("enriching", "Enriching documents...") : t("rescanTooltip", "Rescan workspace")} + + - setCreateOpen(true)} disabled={!selectedAgent}> + setCreateOpen(true)}> - {!selectedAgent && {t("selectAgentFirst", "Select an agent first")}} + {/* Enrichment progress bar */} + {enrichment && enrichment.total > 0 && ( + + + + + + {enrichment.running + ? `${t("enriching", "Enriching")} ${enrichment.done}/${enrichment.total}` + : t("enrichComplete", "Enrichment complete")} + + + )} + {/* Graph — takes full remaining height */} }> @@ -166,7 +204,7 @@ export function VaultPage() { onSelectResult={(doc) => { setDetailDoc(doc); setSelectedDocId(doc.id); }} /> )} - {selectedAgent && } + {/* Detail dialog for all document views */} diff --git a/ui/web/src/schemas/vault.schema.ts b/ui/web/src/schemas/vault.schema.ts index d335f72a..95c23d3e 100644 --- a/ui/web/src/schemas/vault.schema.ts +++ b/ui/web/src/schemas/vault.schema.ts @@ -1,17 +1,9 @@ import { z } from "zod"; -export const vaultCreateSchema = z.object({ - title: z.string().min(1, "Required"), - path: z.string().min(1, "Required"), - docType: z.string().min(1), - scope: z.string().min(1), -}); - export const vaultLinkSchema = z.object({ toDocId: z.string().min(1, "Required"), linkType: z.string().min(1, "Required"), context: z.string().max(500).optional(), }); -export type VaultCreateFormData = z.infer; export type VaultLinkFormData = z.infer; diff --git a/ui/web/src/stores/use-chat-messages-store.ts b/ui/web/src/stores/use-chat-messages-store.ts new file mode 100644 index 00000000..bc271a9d --- /dev/null +++ b/ui/web/src/stores/use-chat-messages-store.ts @@ -0,0 +1,101 @@ +import { create } from "zustand"; +import type { ChatMessage } from "@/types/chat"; + +const MAX_CACHED_SESSIONS = 25; + +interface SessionMessages { + messages: ChatMessage[]; + streamText: string | null; + thinkingText: string | null; + isRunning: boolean; + lastAccessedAt: number; +} + +interface ChatMessagesState { + sessions: Record; + setSessionMessages: (sessionKey: string, messages: ChatMessage[]) => void; + updateSessionMessages: (sessionKey: string, updater: (prev: ChatMessage[]) => ChatMessage[]) => void; + setSessionStream: (sessionKey: string, streamText: string | null) => void; + setSessionThinking: (sessionKey: string, thinkingText: string | null) => void; + setSessionRunning: (sessionKey: string, isRunning: boolean) => void; +} + +// Evict oldest idle sessions when cache exceeds limit. +// Never evict sessions with an active run (isRunning). +function evictStale(sessions: Record): Record { + const keys = Object.keys(sessions); + if (keys.length <= MAX_CACHED_SESSIONS) return sessions; + + const evictable = keys + .filter((k) => !sessions[k]?.isRunning) + .sort((a, b) => (sessions[a]?.lastAccessedAt ?? 0) - (sessions[b]?.lastAccessedAt ?? 0)); + + const toRemove = evictable.slice(0, keys.length - MAX_CACHED_SESSIONS); + if (toRemove.length === 0) return sessions; + + const next = { ...sessions }; + for (const k of toRemove) delete next[k]; + return next; +} + +function touchSession(existing: SessionMessages | undefined, patch: Partial): SessionMessages { + return { + messages: patch.messages ?? existing?.messages ?? [], + streamText: patch.streamText !== undefined ? patch.streamText : (existing?.streamText ?? null), + thinkingText: patch.thinkingText !== undefined ? patch.thinkingText : (existing?.thinkingText ?? null), + isRunning: patch.isRunning !== undefined ? patch.isRunning : (existing?.isRunning ?? false), + lastAccessedAt: Date.now(), + }; +} + +export const useChatMessagesStore = create((set) => ({ + sessions: {}, + + setSessionMessages: (sessionKey, messages) => { + set((state) => ({ + sessions: evictStale({ + ...state.sessions, + [sessionKey]: touchSession(state.sessions[sessionKey], { messages }), + }), + })); + }, + + updateSessionMessages: (sessionKey, updater) => { + set((state) => { + const current = state.sessions[sessionKey]?.messages ?? []; + return { + sessions: evictStale({ + ...state.sessions, + [sessionKey]: touchSession(state.sessions[sessionKey], { messages: updater(current) }), + }), + }; + }); + }, + + setSessionStream: (sessionKey, streamText) => { + set((state) => ({ + sessions: { + ...state.sessions, + [sessionKey]: touchSession(state.sessions[sessionKey], { streamText }), + }, + })); + }, + + setSessionThinking: (sessionKey, thinkingText) => { + set((state) => ({ + sessions: { + ...state.sessions, + [sessionKey]: touchSession(state.sessions[sessionKey], { thinkingText }), + }, + })); + }, + + setSessionRunning: (sessionKey, isRunning) => { + set((state) => ({ + sessions: { + ...state.sessions, + [sessionKey]: touchSession(state.sessions[sessionKey], { isRunning }), + }, + })); + }, +})); diff --git a/ui/web/src/types/agent.ts b/ui/web/src/types/agent.ts index 6dd89160..038f491f 100644 --- a/ui/web/src/types/agent.ts +++ b/ui/web/src/types/agent.ts @@ -67,6 +67,20 @@ export interface MemoryConfig { vector_weight?: number; text_weight?: number; min_score?: number; + dreaming?: DreamingConfig | null; +} + +/** + * DreamingConfig mirrors Go internal/config.DreamingConfig — per-agent + * overrides for the consolidation dreaming worker. Pointer fields stay + * undefined when the operator hasn't set them so partial overrides merge + * cleanly with backend defaults. + */ +export interface DreamingConfig { + enabled?: boolean; + debounce_ms?: number; + threshold?: number; + verbose_log?: boolean; } export interface WorkspaceSharingConfig { diff --git a/ui/web/src/types/evolution.ts b/ui/web/src/types/evolution.ts index 9150e73e..0637a705 100644 --- a/ui/web/src/types/evolution.ts +++ b/ui/web/src/types/evolution.ts @@ -18,7 +18,7 @@ export interface RetrievalAggregate { export interface EvolutionSuggestion { id: string; agent_id: string; - suggestion_type: "threshold" | "tool_order" | "skill_add" | "memory_prune"; + suggestion_type: "threshold" | "tool_order" | "skill_add"; suggestion: string; rationale: string; parameters: Record | null; diff --git a/ui/web/src/types/vault.ts b/ui/web/src/types/vault.ts index c725c597..9ea805f5 100644 --- a/ui/web/src/types/vault.ts +++ b/ui/web/src/types/vault.ts @@ -2,7 +2,7 @@ export interface VaultDocument { id: string; tenant_id: string; - agent_id: string; + agent_id?: string | null; team_id?: string; scope: "personal" | "team" | "shared"; custom_scope?: string;
{t(`${ds}.description`)}
{t("detail.evolution.noMetrics")}
{errors.title.message}
{errors.path.message}
{t("upload.dropzone", "Drop files here or click to browse")}
{t("upload.dropzoneHint", "Text files only (.md, .txt, .json, .yaml, .csv, ...)")}
{t("detail.binaryFile")}
{(doc.metadata?.mime_type as string) || doc.path.split(".").pop()?.toUpperCase()}
{doc.summary}