mirror of
https://github.com/tiennm99/goclaw.git
synced 2026-10-15 08:17:09 +00:00
feat(vault): workspace rescan endpoint with symlink-safe walker
Add POST /v1/agents/{agentID}/vault/rescan to backfill vault_documents
from filesystem. Walks agent workspace, registers missing/changed files,
publishes enrichment events for async summarize + embed + link classify.
Backend:
- SafeWalkWorkspace: symlink-safe walker with exclusion patterns,
resource limits (5K files, 500MB, 50MB/file), context deadline
- RescanWorkspace: hash-based dedup, path-based scope detection
(teams/{id}/ → team scope), idempotent via UpsertDocument ON CONFLICT
- Per-agent mutex (409 Conflict for concurrent rescans)
- Tenant-scoped workspace resolution (config.TenantWorkspace)
- Enrichment worker: semaphore-based parallel summarize (max 3 concurrent)
- Media files without summary skip LLM summarize, embed title+path only
- DRY: export InferTitle/InferDocType from vault pkg, remove from interceptor
- Auto-register text uploads in vault via onTextUploaded callback
Frontend:
- Rescan button in vault page header (FolderSync icon)
- Toast with result counts, 409 warning, error handling
- i18n strings for en/vi/zh
Security: skip all symlinks, boundary checks, per-file size limit,
tenant isolation via server-side workspace resolution.
This commit is contained in:
1 parent
79b5dba5c7
commit
dabb1eaa11
21 files changed
+992
-84
No files matched your search
@@ -202,6 +202,8 @@ func runGateway() {
|
||||
Provider: consolidationProvider,
|
||||
Model: consolidationProvider.DefaultModel(),
|
||||
Extractor: kgExtractor,
|
||||
// Per-agent dreaming overrides (MemoryConfig.Dreaming JSONB).
|
||||
AgentStore: pgStores.Agents,
|
||||
})
|
||||
defer cleanupConsolidation()
|
||||
slog.Info("consolidation pipeline registered")
|
||||
@@ -301,6 +303,7 @@ func runGateway() {
|
||||
skillsLoader: skillsLoader,
|
||||
workspace: workspace,
|
||||
dataDir: dataDir,
|
||||
domainBus: domainBus,
|
||||
}
|
||||
|
||||
gatewayAddr := loopbackAddr(cfg.Gateway.Host, cfg.Gateway.Port)
|
||||
|
||||
@@ -6,6 +6,7 @@ import (
|
||||
"github.com/nextlevelbuilder/goclaw/internal/cache"
|
||||
"github.com/nextlevelbuilder/goclaw/internal/channels"
|
||||
"github.com/nextlevelbuilder/goclaw/internal/config"
|
||||
"github.com/nextlevelbuilder/goclaw/internal/eventbus"
|
||||
"github.com/nextlevelbuilder/goclaw/internal/gateway"
|
||||
"github.com/nextlevelbuilder/goclaw/internal/providers"
|
||||
"github.com/nextlevelbuilder/goclaw/internal/skills"
|
||||
@@ -28,4 +29,5 @@ type gatewayDeps struct {
|
||||
permCache *cache.PermissionCache // nil if no tenant store; closed on shutdown to stop sweep goroutines
|
||||
workspace string
|
||||
dataDir string
|
||||
domainBus eventbus.DomainEventBus
|
||||
}
|
||||
@@ -175,7 +175,7 @@ func (d *gatewayDeps) wireHTTPHandlersOnServer(
|
||||
|
||||
// V3: Knowledge Vault document API
|
||||
if d.pgStores != nil && d.pgStores.Vault != nil {
|
||||
d.server.SetVaultHandler(httpapi.NewVaultHandler(d.pgStores.Vault, d.pgStores.Teams))
|
||||
d.server.SetVaultHandler(httpapi.NewVaultHandler(d.pgStores.Vault, d.pgStores.Teams, d.workspace, d.domainBus))
|
||||
}
|
||||
|
||||
// V3: Episodic memory summaries API
|
||||
|
||||
@@ -141,6 +141,9 @@ func wireExtras(
|
||||
autoInjector = memorypkg.NewAutoInjector(stores.Episodic, stores.EvolutionMetrics)
|
||||
}
|
||||
|
||||
// vaultIntc is set later by wireVault but captured by closure in OnTextUploaded.
|
||||
var vaultIntc *tools.VaultInterceptor
|
||||
|
||||
resolver := agent.NewManagedResolver(agent.ResolverDeps{
|
||||
AgentStore: stores.Agents,
|
||||
ProviderStore: stores.Providers,
|
||||
@@ -187,6 +190,11 @@ func wireExtras(
|
||||
AutoInjector: autoInjector,
|
||||
EvolutionMetricsStore: stores.EvolutionMetrics,
|
||||
DomainBus: domainBus,
|
||||
OnTextUploaded: func(ctx context.Context, path, content string) {
|
||||
if vaultIntc != nil {
|
||||
vaultIntc.AfterWrite(ctx, path, content)
|
||||
}
|
||||
},
|
||||
OnEvent: func(event agent.AgentEvent) {
|
||||
// Sign /v1/files/ and /v1/media/ URLs in content before delivery.
|
||||
// Sessions store clean paths; signing happens only at delivery time.
|
||||
@@ -342,7 +350,7 @@ func wireExtras(
|
||||
}
|
||||
|
||||
// Wire vault tools and interceptors (conditional on vault store availability)
|
||||
wireVault(stores, toolsReg, workspace, domainBus)
|
||||
vaultIntc = wireVault(stores, toolsReg, workspace, domainBus)
|
||||
|
||||
// Wire delegate tool for inter-agent delegation via agent_links.
|
||||
if stores.AgentLinks != nil && stores.Agents != nil {
|
||||
|
||||
@@ -12,9 +12,12 @@ import (
|
||||
// wireVault wires Knowledge Vault tools and interceptors into the tool registry.
|
||||
// All wiring is skipped if stores.Vault is nil.
|
||||
// Pattern mirrors wireExtras KG wiring: register tools, set stores, set interceptors.
|
||||
func wireVault(stores *store.Stores, toolsReg *tools.Registry, workspace string, bus eventbus.DomainEventBus) {
|
||||
// wireVault wires Knowledge Vault tools and interceptors into the tool registry.
|
||||
// Returns the shared VaultInterceptor for use by other subsystems (e.g. agent upload hook).
|
||||
// Returns nil if stores.Vault is nil.
|
||||
func wireVault(stores *store.Stores, toolsReg *tools.Registry, workspace string, bus eventbus.DomainEventBus) *tools.VaultInterceptor {
|
||||
if stores.Vault == nil {
|
||||
return
|
||||
return nil
|
||||
}
|
||||
|
||||
// Register vault tools — these are always available when vault store is present.
|
||||
@@ -71,4 +74,5 @@ func wireVault(stores *store.Stores, toolsReg *tools.Registry, workspace string,
|
||||
}
|
||||
|
||||
slog.Info("vault tools registered", "tools", "vault_search,create_image,create_video,create_audio,tts,edit")
|
||||
return vaultIntc
|
||||
}
|
||||
Reference in new issue
Block a user