mirror of
https://github.com/tiennm99/goclaw.git
synced 2026-10-16 00:14:57 +00:00
feat(kg): add tsvector FTS, entity deduplication, and embedding fixes
- Replace ILIKE search with PostgreSQL tsvector/GIN full-text search for KG entities (migration 000031) - Add entity deduplication system with dual-threshold strategy: auto-merge at 0.98+ similarity with Jaro-Winkler name match, flag 0.90-0.98 as candidates for manual review - Add ScanDuplicates for bulk on-demand duplicate detection - Add MergeEntities with advisory lock and tenant-scoped relation re-pointing (delete-then-update to avoid ON CONFLICT on UPDATE) - Wire dedup inline after KG extraction pipeline - Fix BackfillKGEmbeddings: was failing silently due to context.Background() missing tenant_id; now runs cross-tenant - Fix BackfillKGEmbeddings: break on error → continue with failed ID tracking and max consecutive error cap - Add EmbedEntity helper; UpsertEntity now generates embeddings in background goroutine - Add HTTP endpoints: POST /kg/dedup/scan, GET /kg/dedup, POST /kg/merge, POST /kg/dedup/dismiss - Add web UI: Dedup dialog with Scan All button, side-by-side entity comparison, merge/dismiss actions - Add Jaro-Winkler similarity algorithm with 34 unit tests - Update IngestExtraction to return upserted entity IDs - Bump RequiredSchemaVersion to 31
This commit is contained in:
1 parent
97f4942943
commit
2e869f4ece
22 files changed
+1693
-57
No files matched your search
+12
-1
@@ -583,11 +583,22 @@ func buildKGExtractFunc(kgStore store.KnowledgeGraphStore, bts store.BuiltinTool
|
||||
result.Relations[i].AgentID = agentID
|
||||
result.Relations[i].UserID = userID
|
||||
}
|
||||
if err := kgStore.IngestExtraction(ctx, agentID, userID, result.Entities, result.Relations); err != nil {
|
||||
entityIDs, err := kgStore.IngestExtraction(ctx, agentID, userID, result.Entities, result.Relations)
|
||||
if err != nil {
|
||||
slog.Warn("kg extract: ingest failed", "agent", agentID, "error", err)
|
||||
return
|
||||
}
|
||||
slog.Info("kg extract: ingested from memory write", "agent", agentID, "entities", len(result.Entities), "relations", len(result.Relations))
|
||||
|
||||
// Run inline dedup on newly upserted entities (best-effort, non-blocking)
|
||||
if len(entityIDs) > 0 {
|
||||
merged, flagged, dedupErr := kgStore.DedupAfterExtraction(ctx, agentID, userID, entityIDs)
|
||||
if dedupErr != nil {
|
||||
slog.Warn("kg extract: dedup failed", "agent", agentID, "error", dedupErr)
|
||||
} else if merged > 0 || flagged > 0 {
|
||||
slog.Info("kg extract: dedup completed", "agent", agentID, "auto_merged", merged, "candidates_flagged", flagged)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user