mirror of
https://github.com/tiennm99/goclaw.git
synced 2026-10-19 00:15:05 +00:00
viettranx
2e869f4ece
- Replace ILIKE search with PostgreSQL tsvector/GIN full-text search for KG entities (migration 000031) - Add entity deduplication system with dual-threshold strategy: auto-merge at 0.98+ similarity with Jaro-Winkler name match, flag 0.90-0.98 as candidates for manual review - Add ScanDuplicates for bulk on-demand duplicate detection - Add MergeEntities with advisory lock and tenant-scoped relation re-pointing (delete-then-update to avoid ON CONFLICT on UPDATE) - Wire dedup inline after KG extraction pipeline - Fix BackfillKGEmbeddings: was failing silently due to context.Background() missing tenant_id; now runs cross-tenant - Fix BackfillKGEmbeddings: break on error → continue with failed ID tracking and max consecutive error cap - Add EmbedEntity helper; UpsertEntity now generates embeddings in background goroutine - Add HTTP endpoints: POST /kg/dedup/scan, GET /kg/dedup, POST /kg/merge, POST /kg/dedup/dismiss - Add web UI: Dedup dialog with Scan All button, side-by-side entity comparison, merge/dismiss actions - Add Jaro-Winkler similarity algorithm with 34 unit tests - Update IngestExtraction to return upserted entity IDs - Bump RequiredSchemaVersion to 31
22 lines
1.0 KiB
SQL
22 lines
1.0 KiB
SQL
-- tsvector full-text search for KG entities (replaces ILIKE)
|
|
ALTER TABLE kg_entities ADD COLUMN IF NOT EXISTS tsv tsvector
|
|
GENERATED ALWAYS AS (to_tsvector('simple', name || ' ' || COALESCE(description, ''))) STORED;
|
|
|
|
CREATE INDEX IF NOT EXISTS idx_kg_entities_tsv ON kg_entities USING GIN (tsv);
|
|
|
|
-- Dedup candidates table for entity deduplication review
|
|
CREATE TABLE IF NOT EXISTS kg_dedup_candidates (
|
|
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
|
|
tenant_id UUID REFERENCES tenants(id) ON DELETE CASCADE,
|
|
agent_id UUID NOT NULL REFERENCES agents(id) ON DELETE CASCADE,
|
|
user_id VARCHAR(255) NOT NULL DEFAULT '',
|
|
entity_a_id UUID NOT NULL REFERENCES kg_entities(id) ON DELETE CASCADE,
|
|
entity_b_id UUID NOT NULL REFERENCES kg_entities(id) ON DELETE CASCADE,
|
|
similarity FLOAT NOT NULL,
|
|
status VARCHAR(20) NOT NULL DEFAULT 'pending',
|
|
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
|
UNIQUE(entity_a_id, entity_b_id)
|
|
);
|
|
|
|
CREATE INDEX IF NOT EXISTS idx_kg_dedup_agent ON kg_dedup_candidates(agent_id, status);
|