feat(vault): workspace rescan endpoint with symlink-safe walker

Add POST /v1/agents/{agentID}/vault/rescan to backfill vault_documents
from filesystem. Walks agent workspace, registers missing/changed files,
publishes enrichment events for async summarize + embed + link classify.

Backend:
- SafeWalkWorkspace: symlink-safe walker with exclusion patterns,
  resource limits (5K files, 500MB, 50MB/file), context deadline
- RescanWorkspace: hash-based dedup, path-based scope detection
  (teams/{id}/ → team scope), idempotent via UpsertDocument ON CONFLICT
- Per-agent mutex (409 Conflict for concurrent rescans)
- Tenant-scoped workspace resolution (config.TenantWorkspace)
- Enrichment worker: semaphore-based parallel summarize (max 3 concurrent)
- Media files without summary skip LLM summarize, embed title+path only
- DRY: export InferTitle/InferDocType from vault pkg, remove from interceptor
- Auto-register text uploads in vault via onTextUploaded callback

Frontend:
- Rescan button in vault page header (FolderSync icon)
- Toast with result counts, 409 warning, error handling
- i18n strings for en/vi/zh

Security: skip all symlinks, boundary checks, per-file size limit,
tenant isolation via server-side workspace resolution.
This commit is contained in:
viettranx committed 2026-04-10 13:28:44 +07:00
1 parent 79b5dba5c7
commit dabb1eaa11
21 files changed
+992 -84

No files matched your search

+3
View File
@@ -202,6 +202,8 @@ func runGateway() {
Provider: consolidationProvider,
Model: consolidationProvider.DefaultModel(),
Extractor: kgExtractor,
// Per-agent dreaming overrides (MemoryConfig.Dreaming JSONB).
AgentStore: pgStores.Agents,
})
defer cleanupConsolidation()
slog.Info("consolidation pipeline registered")
@@ -301,6 +303,7 @@ func runGateway() {
skillsLoader: skillsLoader,
workspace: workspace,
dataDir: dataDir,
domainBus: domainBus,
}
gatewayAddr := loopbackAddr(cfg.Gateway.Host, cfg.Gateway.Port)
+2
View File
@@ -6,6 +6,7 @@ import (
"github.com/nextlevelbuilder/goclaw/internal/cache"
"github.com/nextlevelbuilder/goclaw/internal/channels"
"github.com/nextlevelbuilder/goclaw/internal/config"
"github.com/nextlevelbuilder/goclaw/internal/eventbus"
"github.com/nextlevelbuilder/goclaw/internal/gateway"
"github.com/nextlevelbuilder/goclaw/internal/providers"
"github.com/nextlevelbuilder/goclaw/internal/skills"
@@ -28,4 +29,5 @@ type gatewayDeps struct {
permCache *cache.PermissionCache // nil if no tenant store; closed on shutdown to stop sweep goroutines
workspace string
dataDir string
domainBus eventbus.DomainEventBus
}
+1 -1
View File
@@ -175,7 +175,7 @@ func (d *gatewayDeps) wireHTTPHandlersOnServer(
// V3: Knowledge Vault document API
if d.pgStores != nil && d.pgStores.Vault != nil {
d.server.SetVaultHandler(httpapi.NewVaultHandler(d.pgStores.Vault, d.pgStores.Teams))
d.server.SetVaultHandler(httpapi.NewVaultHandler(d.pgStores.Vault, d.pgStores.Teams, d.workspace, d.domainBus))
}
// V3: Episodic memory summaries API
+9 -1
View File
@@ -141,6 +141,9 @@ func wireExtras(
autoInjector = memorypkg.NewAutoInjector(stores.Episodic, stores.EvolutionMetrics)
}
// vaultIntc is set later by wireVault but captured by closure in OnTextUploaded.
var vaultIntc *tools.VaultInterceptor
resolver := agent.NewManagedResolver(agent.ResolverDeps{
AgentStore: stores.Agents,
ProviderStore: stores.Providers,
@@ -187,6 +190,11 @@ func wireExtras(
AutoInjector: autoInjector,
EvolutionMetricsStore: stores.EvolutionMetrics,
DomainBus: domainBus,
OnTextUploaded: func(ctx context.Context, path, content string) {
if vaultIntc != nil {
vaultIntc.AfterWrite(ctx, path, content)
}
},
OnEvent: func(event agent.AgentEvent) {
// Sign /v1/files/ and /v1/media/ URLs in content before delivery.
// Sessions store clean paths; signing happens only at delivery time.
@@ -342,7 +350,7 @@ func wireExtras(
}
// Wire vault tools and interceptors (conditional on vault store availability)
wireVault(stores, toolsReg, workspace, domainBus)
vaultIntc = wireVault(stores, toolsReg, workspace, domainBus)
// Wire delegate tool for inter-agent delegation via agent_links.
if stores.AgentLinks != nil && stores.Agents != nil {
+6 -2
View File
@@ -12,9 +12,12 @@ import (
// wireVault wires Knowledge Vault tools and interceptors into the tool registry.
// All wiring is skipped if stores.Vault is nil.
// Pattern mirrors wireExtras KG wiring: register tools, set stores, set interceptors.
func wireVault(stores *store.Stores, toolsReg *tools.Registry, workspace string, bus eventbus.DomainEventBus) {
// wireVault wires Knowledge Vault tools and interceptors into the tool registry.
// Returns the shared VaultInterceptor for use by other subsystems (e.g. agent upload hook).
// Returns nil if stores.Vault is nil.
func wireVault(stores *store.Stores, toolsReg *tools.Registry, workspace string, bus eventbus.DomainEventBus) *tools.VaultInterceptor {
if stores.Vault == nil {
return
return nil
}
// Register vault tools — these are always available when vault store is present.
@@ -71,4 +74,5 @@ func wireVault(stores *store.Stores, toolsReg *tools.Registry, workspace string,
}
slog.Info("vault tools registered", "tools", "vault_search,create_image,create_video,create_audio,tts,edit")
return vaultIntc
}