mirror of
https://github.com/tiennm99/goclaw.git
synced 2026-10-15 18:15:47 +00:00
feat(kg): add tsvector FTS, entity deduplication, and embedding fixes
- Replace ILIKE search with PostgreSQL tsvector/GIN full-text search for KG entities (migration 000031) - Add entity deduplication system with dual-threshold strategy: auto-merge at 0.98+ similarity with Jaro-Winkler name match, flag 0.90-0.98 as candidates for manual review - Add ScanDuplicates for bulk on-demand duplicate detection - Add MergeEntities with advisory lock and tenant-scoped relation re-pointing (delete-then-update to avoid ON CONFLICT on UPDATE) - Wire dedup inline after KG extraction pipeline - Fix BackfillKGEmbeddings: was failing silently due to context.Background() missing tenant_id; now runs cross-tenant - Fix BackfillKGEmbeddings: break on error → continue with failed ID tracking and max consecutive error cap - Add EmbedEntity helper; UpsertEntity now generates embeddings in background goroutine - Add HTTP endpoints: POST /kg/dedup/scan, GET /kg/dedup, POST /kg/merge, POST /kg/dedup/dismiss - Add web UI: Dedup dialog with Scan All button, side-by-side entity comparison, merge/dismiss actions - Add Jaro-Winkler similarity algorithm with 34 unit tests - Update IngestExtraction to return upserted entity IDs - Bump RequiredSchemaVersion to 31
This commit is contained in:
1 parent
97f4942943
commit
2e869f4ece
22 files changed
+1693
-57
No files matched your search
@@ -89,14 +89,34 @@ func (m *mockKGStore) Traverse(_ context.Context, _, _, startEntityID string, _
|
||||
return m.traversal[startEntityID], nil
|
||||
}
|
||||
|
||||
func (m *mockKGStore) IngestExtraction(context.Context, string, string, []store.Entity, []store.Relation) error {
|
||||
return nil
|
||||
func (m *mockKGStore) IngestExtraction(context.Context, string, string, []store.Entity, []store.Relation) ([]string, error) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func (m *mockKGStore) PruneByConfidence(context.Context, string, string, float64) (int, error) {
|
||||
return 0, nil
|
||||
}
|
||||
|
||||
func (m *mockKGStore) DedupAfterExtraction(context.Context, string, string, []string) (int, int, error) {
|
||||
return 0, 0, nil
|
||||
}
|
||||
|
||||
func (m *mockKGStore) ScanDuplicates(context.Context, string, string, float64, int) (int, error) {
|
||||
return 0, nil
|
||||
}
|
||||
|
||||
func (m *mockKGStore) ListDedupCandidates(context.Context, string, string, int) ([]store.DedupCandidate, error) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func (m *mockKGStore) MergeEntities(context.Context, string, string, string, string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *mockKGStore) DismissCandidate(context.Context, string, string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *mockKGStore) Stats(context.Context, string, string) (*store.GraphStats, error) {
|
||||
return &store.GraphStats{}, nil
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user