mirror of
https://github.com/tiennm99/goclaw.git
synced 2026-10-14 02:15:57 +00:00
refactor(vault): remove vault_link/vault_backlinks tools, auto-sync wikilinks in enrichment pipeline
- Remove vault_link and vault_backlinks builtin tools (replaced by auto-linking)
- Add DeleteDocLinksByType to VaultStore interface (PG + SQLite) to selectively
delete links by type without destroying semantic links
- Integrate SyncDocLinks into enrichment worker: [[wikilinks]] are now
auto-extracted from document content and synced as vault links on every write
- Fix SyncDocLinks to use DeleteDocLinksByType("wikilink") instead of
DeleteDocLinks which was deleting all link types including semantic
- Add missing coreToolSummaries for delegate, memory_expand, vault_search
(previously showed as "(custom tool)" in system prompt)
- Increase enrichSimilarityLimit from 5 to 10 for richer auto-linking
This commit is contained in:
1 parent
25a43411ff
commit
bb365a66c4
8 files changed
+57
-247
No files matched your search
@@ -19,15 +19,7 @@ func wireVault(stores *store.Stores, toolsReg *tools.Registry, workspace string,
|
||||
|
||||
// Register vault tools — these are always available when vault store is present.
|
||||
vaultSearchTool := tools.NewVaultSearchTool()
|
||||
vaultLinkTool := tools.NewVaultLinkTool()
|
||||
vaultBacklinksTool := tools.NewVaultBacklinksTool()
|
||||
toolsReg.Register(vaultSearchTool)
|
||||
toolsReg.Register(vaultLinkTool)
|
||||
toolsReg.Register(vaultBacklinksTool)
|
||||
|
||||
// Wire vault store onto link/backlinks tools.
|
||||
vaultLinkTool.SetVaultStore(stores.Vault)
|
||||
vaultBacklinksTool.SetVaultStore(stores.Vault)
|
||||
|
||||
// Build VaultSearchService: fan-out across vault + KG (episodic store pending impl).
|
||||
// EpisodicStore is nil until a PG implementation exists.
|
||||
@@ -78,5 +70,5 @@ func wireVault(stores *store.Stores, toolsReg *tools.Registry, workspace string,
|
||||
}
|
||||
}
|
||||
|
||||
slog.Info("vault tools registered", "tools", "vault_search,vault_link,vault_backlinks,create_image,create_video,create_audio,tts,edit")
|
||||
slog.Info("vault tools registered", "tools", "vault_search,create_image,create_video,create_audio,tts,edit")
|
||||
}
|
||||
@@ -205,6 +205,9 @@ var coreToolSummaries = map[string]string{
|
||||
"team_tasks": "Team task board — track progress, manage dependencies (spawn auto-creates delegation tasks)",
|
||||
"list_group_members": "List all members of the current group chat (Feishu/Lark only)",
|
||||
"create_forum_topic": "Create a forum topic in a Telegram supergroup",
|
||||
"delegate": "Delegate a task to a linked agent (requires agent_links). See ## Delegation Targets for available agents",
|
||||
"memory_expand": "Retrieve full session details from episodic memory results — use after memory_search returns episodic hits",
|
||||
"vault_search": "Search documents in the knowledge vault (hybrid keyword + semantic)",
|
||||
|
||||
// Tool aliases (edit_file, sessions_spawn, Read, Write, Edit, Bash, etc.)
|
||||
// are registered in the tool registry but excluded from the system prompt
|
||||
|
||||
@@ -132,6 +132,19 @@ func (s *PGVaultStore) DeleteDocLinks(ctx context.Context, tenantID, docID strin
|
||||
return err
|
||||
}
|
||||
|
||||
// DeleteDocLinksByType removes outbound links of a specific type from a document.
|
||||
func (s *PGVaultStore) DeleteDocLinksByType(ctx context.Context, tenantID, docID, linkType string) error {
|
||||
uid := mustParseUUID(docID)
|
||||
tid := mustParseUUID(tenantID)
|
||||
_, err := s.db.ExecContext(ctx, `
|
||||
DELETE FROM vault_links vl
|
||||
USING vault_documents vd
|
||||
WHERE vl.from_doc_id = $1
|
||||
AND vd.id = vl.from_doc_id AND vd.tenant_id = $2
|
||||
AND vl.link_type = $3`, uid, tid, linkType)
|
||||
return err
|
||||
}
|
||||
|
||||
func scanVaultLinks(rows *sql.Rows) ([]store.VaultLink, error) {
|
||||
var links []store.VaultLink
|
||||
for rows.Next() {
|
||||
|
||||
@@ -111,6 +111,17 @@ func (s *SQLiteVaultStore) DeleteDocLinks(ctx context.Context, tenantID, docID s
|
||||
return err
|
||||
}
|
||||
|
||||
// DeleteDocLinksByType removes outbound links of a specific type from a document, scoped by tenant.
|
||||
func (s *SQLiteVaultStore) DeleteDocLinksByType(ctx context.Context, tenantID, docID, linkType string) error {
|
||||
_, err := s.db.ExecContext(ctx, `
|
||||
DELETE FROM vault_links
|
||||
WHERE from_doc_id = ?
|
||||
AND link_type = ?
|
||||
AND from_doc_id IN (SELECT id FROM vault_documents WHERE tenant_id = ?)`,
|
||||
docID, linkType, tenantID)
|
||||
return err
|
||||
}
|
||||
|
||||
func scanVaultLinkRows(rows *sql.Rows) ([]store.VaultLink, error) {
|
||||
var links []store.VaultLink
|
||||
for rows.Next() {
|
||||
|
||||
@@ -92,6 +92,7 @@ type VaultStore interface {
|
||||
GetOutLinks(ctx context.Context, tenantID, docID string) ([]VaultLink, error)
|
||||
GetBacklinks(ctx context.Context, tenantID, docID string) ([]VaultBacklink, error)
|
||||
DeleteDocLinks(ctx context.Context, tenantID, docID string) error
|
||||
DeleteDocLinksByType(ctx context.Context, tenantID, docID, linkType string) error
|
||||
|
||||
// Enrichment
|
||||
// UpdateSummaryAndReembed updates summary text and re-generates embedding from title+path+summary.
|
||||
|
||||
@@ -1,233 +0,0 @@
|
||||
package tools
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"github.com/google/uuid"
|
||||
|
||||
"github.com/nextlevelbuilder/goclaw/internal/store"
|
||||
)
|
||||
|
||||
// VaultLinkTool creates explicit links between vault documents.
|
||||
type VaultLinkTool struct {
|
||||
vaultStore store.VaultStore
|
||||
}
|
||||
|
||||
func NewVaultLinkTool() *VaultLinkTool {
|
||||
return &VaultLinkTool{}
|
||||
}
|
||||
|
||||
func (t *VaultLinkTool) SetVaultStore(vs store.VaultStore) {
|
||||
t.vaultStore = vs
|
||||
}
|
||||
|
||||
func (t *VaultLinkTool) Name() string { return "vault_link" }
|
||||
|
||||
func (t *VaultLinkTool) Description() string {
|
||||
return "Create an explicit link between two vault documents. Similar to [[wikilinks]] in markdown."
|
||||
}
|
||||
|
||||
func (t *VaultLinkTool) Parameters() map[string]any {
|
||||
return map[string]any{
|
||||
"type": "object",
|
||||
"properties": map[string]any{
|
||||
"from": map[string]any{
|
||||
"type": "string",
|
||||
"description": "Source document path (workspace-relative)",
|
||||
},
|
||||
"to": map[string]any{
|
||||
"type": "string",
|
||||
"description": "Target document path (workspace-relative)",
|
||||
},
|
||||
"context": map[string]any{
|
||||
"type": "string",
|
||||
"description": "Optional note describing the link relationship",
|
||||
},
|
||||
"link_type": map[string]any{
|
||||
"type": "string",
|
||||
"description": "Link type: wikilink (default) or reference",
|
||||
"enum": []string{"wikilink", "reference"},
|
||||
},
|
||||
},
|
||||
"required": []string{"from", "to"},
|
||||
}
|
||||
}
|
||||
|
||||
func (t *VaultLinkTool) Execute(ctx context.Context, args map[string]any) *Result {
|
||||
fromPath, _ := args["from"].(string)
|
||||
toPath, _ := args["to"].(string)
|
||||
linkCtx, _ := args["context"].(string)
|
||||
linkType, _ := args["link_type"].(string)
|
||||
if linkType == "" {
|
||||
linkType = "wikilink"
|
||||
}
|
||||
|
||||
if fromPath == "" || toPath == "" {
|
||||
return ErrorResult("both 'from' and 'to' paths are required")
|
||||
}
|
||||
if linkType != "wikilink" && linkType != "reference" {
|
||||
return ErrorResult("link_type must be 'wikilink' or 'reference'")
|
||||
}
|
||||
|
||||
agentID := store.AgentIDFromContext(ctx)
|
||||
tenantID := store.TenantIDFromContext(ctx)
|
||||
if t.vaultStore == nil || agentID == uuid.Nil {
|
||||
return ErrorResult("vault not available")
|
||||
}
|
||||
|
||||
tid := tenantID.String()
|
||||
aid := agentID.String()
|
||||
|
||||
// Infer scope and team from context.
|
||||
var teamID *string
|
||||
scope := "personal"
|
||||
if rc := store.RunContextFromCtx(ctx); rc != nil && rc.TeamID != "" {
|
||||
teamID = &rc.TeamID
|
||||
scope = "team"
|
||||
}
|
||||
|
||||
// Resolve source doc (auto-register if not in vault)
|
||||
fromDoc, err := t.resolveOrRegister(ctx, tid, aid, teamID, scope, fromPath)
|
||||
if err != nil {
|
||||
return ErrorResult(fmt.Sprintf("cannot resolve source doc: %v", err))
|
||||
}
|
||||
|
||||
// Resolve target doc (auto-register if not in vault)
|
||||
toDoc, err := t.resolveOrRegister(ctx, tid, aid, teamID, scope, toPath)
|
||||
if err != nil {
|
||||
return ErrorResult(fmt.Sprintf("cannot resolve target doc: %v", err))
|
||||
}
|
||||
|
||||
// Block cross-team links (both team docs must be same team).
|
||||
if fromDoc.TeamID != nil && toDoc.TeamID != nil && *fromDoc.TeamID != *toDoc.TeamID {
|
||||
return ErrorResult("cannot link documents from different teams")
|
||||
}
|
||||
|
||||
link := &store.VaultLink{
|
||||
FromDocID: fromDoc.ID,
|
||||
ToDocID: toDoc.ID,
|
||||
LinkType: linkType,
|
||||
Context: linkCtx,
|
||||
}
|
||||
if err := t.vaultStore.CreateLink(ctx, link); err != nil {
|
||||
return ErrorResult(fmt.Sprintf("failed to create link: %v", err))
|
||||
}
|
||||
|
||||
return NewResult(fmt.Sprintf("Linked %s → %s", fromPath, toPath))
|
||||
}
|
||||
|
||||
// resolveOrRegister finds a vault doc by path, or creates a stub entry with team context.
|
||||
func (t *VaultLinkTool) resolveOrRegister(ctx context.Context, tenantID, agentID string, teamID *string, scope, path string) (*store.VaultDocument, error) {
|
||||
doc, err := t.vaultStore.GetDocument(ctx, tenantID, agentID, path)
|
||||
if err == nil && doc != nil {
|
||||
return doc, nil
|
||||
}
|
||||
// Auto-register stub with team context.
|
||||
doc = &store.VaultDocument{
|
||||
TenantID: tenantID,
|
||||
AgentID: agentID,
|
||||
TeamID: teamID,
|
||||
Scope: scope,
|
||||
Path: path,
|
||||
Title: strings.TrimSuffix(path, ".md"),
|
||||
DocType: inferVaultDocType(path),
|
||||
}
|
||||
if err := t.vaultStore.UpsertDocument(ctx, doc); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return t.vaultStore.GetDocument(ctx, tenantID, agentID, path)
|
||||
}
|
||||
|
||||
// VaultBacklinksTool shows all documents linking to a specific document.
|
||||
type VaultBacklinksTool struct {
|
||||
vaultStore store.VaultStore
|
||||
}
|
||||
|
||||
func NewVaultBacklinksTool() *VaultBacklinksTool {
|
||||
return &VaultBacklinksTool{}
|
||||
}
|
||||
|
||||
func (t *VaultBacklinksTool) SetVaultStore(vs store.VaultStore) {
|
||||
t.vaultStore = vs
|
||||
}
|
||||
|
||||
func (t *VaultBacklinksTool) Name() string { return "vault_backlinks" }
|
||||
|
||||
func (t *VaultBacklinksTool) Description() string {
|
||||
return "Show all documents that link TO this document. Useful for tracing dependencies and references."
|
||||
}
|
||||
|
||||
func (t *VaultBacklinksTool) Parameters() map[string]any {
|
||||
return map[string]any{
|
||||
"type": "object",
|
||||
"properties": map[string]any{
|
||||
"path": map[string]any{
|
||||
"type": "string",
|
||||
"description": "Document path to find backlinks for",
|
||||
},
|
||||
},
|
||||
"required": []string{"path"},
|
||||
}
|
||||
}
|
||||
|
||||
func (t *VaultBacklinksTool) Execute(ctx context.Context, args map[string]any) *Result {
|
||||
path, _ := args["path"].(string)
|
||||
if path == "" {
|
||||
return ErrorResult("path parameter is required")
|
||||
}
|
||||
|
||||
agentID := store.AgentIDFromContext(ctx)
|
||||
tenantID := store.TenantIDFromContext(ctx)
|
||||
if t.vaultStore == nil || agentID == uuid.Nil {
|
||||
return ErrorResult("vault not available")
|
||||
}
|
||||
|
||||
doc, err := t.vaultStore.GetDocument(ctx, tenantID.String(), agentID.String(), path)
|
||||
if err != nil {
|
||||
return ErrorResult(fmt.Sprintf("document not found in vault: %s", path))
|
||||
}
|
||||
|
||||
backlinks, err := t.vaultStore.GetBacklinks(ctx, tenantID.String(), doc.ID)
|
||||
if err != nil {
|
||||
return ErrorResult(fmt.Sprintf("failed to get backlinks: %v", err))
|
||||
}
|
||||
|
||||
// Determine current team context for filtering.
|
||||
var currentTeamID string
|
||||
if rc := store.RunContextFromCtx(ctx); rc != nil {
|
||||
currentTeamID = rc.TeamID
|
||||
}
|
||||
|
||||
var sb strings.Builder
|
||||
sb.WriteString(fmt.Sprintf("Documents linking to %s:\n\n", path))
|
||||
count := 0
|
||||
for _, bl := range backlinks {
|
||||
// Team boundary filter:
|
||||
// - Team context: show same-team docs only (hide personal + other teams)
|
||||
// - Personal context: show personal docs only (hide team docs)
|
||||
if bl.TeamID != nil && *bl.TeamID != "" {
|
||||
if currentTeamID == "" || *bl.TeamID != currentTeamID {
|
||||
continue
|
||||
}
|
||||
} else {
|
||||
// Personal doc — hide in team context (prevents exfiltration)
|
||||
if currentTeamID != "" {
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
count++
|
||||
sb.WriteString(fmt.Sprintf("%d. %s (%s)", count, bl.Title, bl.Path))
|
||||
if bl.Context != "" {
|
||||
sb.WriteString(fmt.Sprintf(" — \"%s\"", bl.Context))
|
||||
}
|
||||
sb.WriteByte('\n')
|
||||
}
|
||||
|
||||
if count == 0 {
|
||||
return NewResult(fmt.Sprintf("No documents link to %s", path))
|
||||
}
|
||||
return NewResult(sb.String())
|
||||
}
|
||||
@@ -19,7 +19,7 @@ const (
|
||||
enrichMaxDedupEntries = 10000
|
||||
enrichContentMaxRunes = 8192
|
||||
enrichLLMTimeout = 5 * time.Minute
|
||||
enrichSimilarityLimit = 5
|
||||
enrichSimilarityLimit = 10
|
||||
enrichSimilarityMin = 0.7
|
||||
)
|
||||
|
||||
@@ -157,6 +157,9 @@ func (w *enrichWorker) processBatch(ctx context.Context, key string) {
|
||||
|
||||
// Auto-link via vector similarity.
|
||||
w.autoLink(ctx, r.payload.TenantID, r.payload.AgentID, r.payload.DocID)
|
||||
|
||||
// Sync [[wikilinks]] from document content (text files only).
|
||||
w.syncWikilinks(ctx, r.payload)
|
||||
}
|
||||
|
||||
if w.queue.TryFinish(key) {
|
||||
@@ -188,6 +191,25 @@ func (w *enrichWorker) summarize(ctx context.Context, path, content string) (str
|
||||
return strings.TrimSpace(resp.Content), nil
|
||||
}
|
||||
|
||||
// syncWikilinks extracts [[wikilinks]] from document content and syncs them as vault links.
|
||||
// Only processes text files; binary/media files are silently skipped.
|
||||
func (w *enrichWorker) syncWikilinks(ctx context.Context, p eventbus.VaultDocUpsertedPayload) {
|
||||
fullPath := filepath.Join(p.Workspace, p.Path)
|
||||
content, err := os.ReadFile(fullPath)
|
||||
if err != nil {
|
||||
return // file may be deleted or moved
|
||||
}
|
||||
|
||||
doc, err := w.vault.GetDocumentByID(ctx, p.TenantID, p.DocID)
|
||||
if err != nil || doc == nil {
|
||||
return
|
||||
}
|
||||
|
||||
if err := SyncDocLinks(ctx, w.vault, doc, string(content), p.TenantID, p.AgentID); err != nil {
|
||||
slog.Warn("vault.enrich: sync_wikilinks", "path", p.Path, "err", err)
|
||||
}
|
||||
}
|
||||
|
||||
// autoLink finds similar documents and creates semantic links.
|
||||
func (w *enrichWorker) autoLink(ctx context.Context, tenantID, agentID, docID string) {
|
||||
neighbors, err := w.vault.FindSimilarDocs(ctx, tenantID, agentID, docID, enrichSimilarityLimit)
|
||||
|
||||
@@ -89,12 +89,13 @@ func ResolveWikilinkTarget(ctx context.Context, vs store.VaultStore, target, ten
|
||||
func SyncDocLinks(ctx context.Context, vs store.VaultStore, doc *store.VaultDocument, content, tenantID, agentID string) error {
|
||||
matches := ExtractWikilinks(content)
|
||||
if len(matches) == 0 {
|
||||
// No links — delete existing outbound links
|
||||
return vs.DeleteDocLinks(ctx, tenantID, doc.ID)
|
||||
// No wikilinks — delete existing outbound wikilink-type links only.
|
||||
return vs.DeleteDocLinksByType(ctx, tenantID, doc.ID, "wikilink")
|
||||
}
|
||||
|
||||
// Delete old outbound links first (replace strategy)
|
||||
if err := vs.DeleteDocLinks(ctx, tenantID, doc.ID); err != nil {
|
||||
// Delete old outbound wikilink-type links first (replace strategy).
|
||||
// Preserves semantic links created by auto-linking.
|
||||
if err := vs.DeleteDocLinksByType(ctx, tenantID, doc.ID, "wikilink"); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
|
||||
Reference in new issue
Block a user