feat(kg): add tsvector FTS, entity deduplication, and embedding fixes

- Replace ILIKE search with PostgreSQL tsvector/GIN full-text search
  for KG entities (migration 000031)
- Add entity deduplication system with dual-threshold strategy:
  auto-merge at 0.98+ similarity with Jaro-Winkler name match,
  flag 0.90-0.98 as candidates for manual review
- Add ScanDuplicates for bulk on-demand duplicate detection
- Add MergeEntities with advisory lock and tenant-scoped relation
  re-pointing (delete-then-update to avoid ON CONFLICT on UPDATE)
- Wire dedup inline after KG extraction pipeline
- Fix BackfillKGEmbeddings: was failing silently due to
  context.Background() missing tenant_id; now runs cross-tenant
- Fix BackfillKGEmbeddings: break on error → continue with failed
  ID tracking and max consecutive error cap
- Add EmbedEntity helper; UpsertEntity now generates embeddings
  in background goroutine
- Add HTTP endpoints: POST /kg/dedup/scan, GET /kg/dedup,
  POST /kg/merge, POST /kg/dedup/dismiss
- Add web UI: Dedup dialog with Scan All button, side-by-side
  entity comparison, merge/dismiss actions
- Add Jaro-Winkler similarity algorithm with 34 unit tests
- Update IngestExtraction to return upserted entity IDs
- Bump RequiredSchemaVersion to 31
This commit is contained in:
viettranx committed 2026-03-29 12:15:21 +07:00
1 parent 97f4942943
commit 2e869f4ece
22 files changed
+1693 -57

No files matched your search

+12 -1
View File
@@ -583,11 +583,22 @@ func buildKGExtractFunc(kgStore store.KnowledgeGraphStore, bts store.BuiltinTool
result.Relations[i].AgentID = agentID
result.Relations[i].UserID = userID
}
if err := kgStore.IngestExtraction(ctx, agentID, userID, result.Entities, result.Relations); err != nil {
entityIDs, err := kgStore.IngestExtraction(ctx, agentID, userID, result.Entities, result.Relations)
if err != nil {
slog.Warn("kg extract: ingest failed", "agent", agentID, "error", err)
return
}
slog.Info("kg extract: ingested from memory write", "agent", agentID, "entities", len(result.Entities), "relations", len(result.Relations))
// Run inline dedup on newly upserted entities (best-effort, non-blocking)
if len(entityIDs) > 0 {
merged, flagged, dedupErr := kgStore.DedupAfterExtraction(ctx, agentID, userID, entityIDs)
if dedupErr != nil {
slog.Warn("kg extract: dedup failed", "agent", agentID, "error", dedupErr)
} else if merged > 0 || flagged > 0 {
slog.Info("kg extract: dedup completed", "agent", agentID, "auto_merged", merged, "candidates_flagged", flagged)
}
}
}
}