mirror of
https://github.com/tiennm99/goclaw.git
synced 2026-10-14 00:16:38 +00:00
fix(ui/graph): use random+strongGravity FA2 for sparse vault graphs
Sparse graphs (edges/node < 0.8) previously used Louvain community detection which created hundreds of singleton communities → grid layout. Now uses random disc init + FA2 with strongGravityMode + higher gravity to pull orphan nodes inward. Dense graphs keep Louvain-seeded layout.
This commit is contained in:
viettranx committed
1 parent
49fced695f
commit
0f79720a2d
9 files changed
+155
-129
No files matched your search
@@ -116,6 +116,7 @@ func (h *VaultHandler) RegisterRoutes(mux *http.ServeMux) {
|
||||
mux.HandleFunc("GET /v1/vault/documents/{docID}/links", h.auth(h.handleGetLinks))
|
||||
mux.HandleFunc("POST /v1/vault/links", h.auth(h.handleCreateLink))
|
||||
mux.HandleFunc("DELETE /v1/vault/links/{linkID}", h.auth(h.handleDeleteLink))
|
||||
mux.HandleFunc("POST /v1/vault/links/batch", h.auth(h.handleBatchGetLinks))
|
||||
mux.HandleFunc("POST /v1/vault/rescan", h.auth(h.handleRescan))
|
||||
mux.HandleFunc("POST /v1/vault/search", h.auth(h.handleSearchAll))
|
||||
mux.HandleFunc("GET /v1/vault/enrichment/status", h.auth(h.handleEnrichmentStatus))
|
||||
@@ -602,6 +603,37 @@ func (h *VaultHandler) handleCreateLink(w http.ResponseWriter, r *http.Request)
|
||||
writeJSON(w, http.StatusCreated, link)
|
||||
}
|
||||
|
||||
// handleBatchGetLinks returns all outlinks for a batch of doc IDs in one query.
|
||||
func (h *VaultHandler) handleBatchGetLinks(w http.ResponseWriter, r *http.Request) {
|
||||
locale := extractLocale(r)
|
||||
tenantID := store.TenantIDFromContext(r.Context())
|
||||
|
||||
var body struct {
|
||||
DocIDs []string `json:"doc_ids"`
|
||||
}
|
||||
if !bindJSON(w, r, locale, &body) {
|
||||
return
|
||||
}
|
||||
if len(body.DocIDs) == 0 {
|
||||
writeJSON(w, http.StatusOK, []store.VaultLink{})
|
||||
return
|
||||
}
|
||||
if len(body.DocIDs) > 500 {
|
||||
body.DocIDs = body.DocIDs[:500]
|
||||
}
|
||||
|
||||
links, err := h.store.GetOutLinksBatch(r.Context(), tenantID.String(), body.DocIDs)
|
||||
if err != nil {
|
||||
slog.Warn("vault.batch_links failed", "error", err)
|
||||
writeJSON(w, http.StatusInternalServerError, map[string]string{"error": err.Error()})
|
||||
return
|
||||
}
|
||||
if links == nil {
|
||||
links = []store.VaultLink{}
|
||||
}
|
||||
writeJSON(w, http.StatusOK, links)
|
||||
}
|
||||
|
||||
// handleDeleteLink deletes a vault link.
|
||||
func (h *VaultHandler) handleDeleteLink(w http.ResponseWriter, r *http.Request) {
|
||||
tenantID := store.TenantIDFromContext(r.Context())
|
||||
|
||||
@@ -88,6 +88,25 @@ func (s *PGVaultStore) GetOutLinks(ctx context.Context, tenantID, docID string)
|
||||
return scanVaultLinks(rows)
|
||||
}
|
||||
|
||||
// GetOutLinksBatch returns all outlinks for multiple doc IDs in a single query.
|
||||
func (s *PGVaultStore) GetOutLinksBatch(ctx context.Context, tenantID string, docIDs []string) ([]store.VaultLink, error) {
|
||||
if len(docIDs) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
tid := mustParseUUID(tenantID)
|
||||
rows, err := s.db.QueryContext(ctx, `
|
||||
SELECT vl.id, vl.from_doc_id, vl.to_doc_id, vl.link_type, vl.context, vl.created_at
|
||||
FROM vault_links vl
|
||||
JOIN vault_documents vd ON vl.from_doc_id = vd.id
|
||||
WHERE vl.from_doc_id = ANY($1) AND vd.tenant_id = $2
|
||||
ORDER BY vl.created_at`, pqStringArray(docIDs), tid)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
return scanVaultLinks(rows)
|
||||
}
|
||||
|
||||
// GetBacklinks returns enriched backlinks pointing to a document (single JOIN, LIMIT 100).
|
||||
func (s *PGVaultStore) GetBacklinks(ctx context.Context, tenantID, docID string) ([]store.VaultBacklink, error) {
|
||||
did := mustParseUUID(docID)
|
||||
|
||||
@@ -78,6 +78,30 @@ func (s *SQLiteVaultStore) GetOutLinks(ctx context.Context, tenantID, docID stri
|
||||
return scanVaultLinkRows(rows)
|
||||
}
|
||||
|
||||
// GetOutLinksBatch returns all outlinks for multiple doc IDs in a single query.
|
||||
func (s *SQLiteVaultStore) GetOutLinksBatch(ctx context.Context, tenantID string, docIDs []string) ([]store.VaultLink, error) {
|
||||
if len(docIDs) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
ph := strings.Repeat("?,", len(docIDs)-1) + "?"
|
||||
args := make([]any, 0, len(docIDs)+1)
|
||||
for _, id := range docIDs {
|
||||
args = append(args, id)
|
||||
}
|
||||
args = append(args, tenantID)
|
||||
rows, err := s.db.QueryContext(ctx,
|
||||
`SELECT vl.id, vl.from_doc_id, vl.to_doc_id, vl.link_type, vl.context, vl.created_at
|
||||
FROM vault_links vl
|
||||
JOIN vault_documents d ON d.id = vl.from_doc_id
|
||||
WHERE vl.from_doc_id IN (`+ph+`) AND d.tenant_id = ?
|
||||
ORDER BY vl.created_at`, args...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
return scanVaultLinkRows(rows)
|
||||
}
|
||||
|
||||
// GetBacklinks returns enriched backlinks pointing to a document (single JOIN, LIMIT 100).
|
||||
func (s *SQLiteVaultStore) GetBacklinks(ctx context.Context, tenantID, docID string) ([]store.VaultBacklink, error) {
|
||||
rows, err := s.db.QueryContext(ctx, `
|
||||
|
||||
@@ -90,6 +90,7 @@ type VaultStore interface {
|
||||
CreateLink(ctx context.Context, link *VaultLink) error
|
||||
DeleteLink(ctx context.Context, tenantID, id string) error
|
||||
GetOutLinks(ctx context.Context, tenantID, docID string) ([]VaultLink, error)
|
||||
GetOutLinksBatch(ctx context.Context, tenantID string, docIDs []string) ([]VaultLink, error)
|
||||
GetBacklinks(ctx context.Context, tenantID, docID string) ([]VaultBacklink, error)
|
||||
DeleteDocLinks(ctx context.Context, tenantID, docID string) error
|
||||
DeleteDocLinksByType(ctx context.Context, tenantID, docID, linkType string) error
|
||||
|
||||
@@ -76,97 +76,87 @@ export function SigmaGraphContainer({
|
||||
useEffect(() => {
|
||||
if (!containerRef.current || graph.order === 0) return;
|
||||
|
||||
// Step 1: Detect communities with Louvain, then seed nodes spatially per community.
|
||||
// This breaks the "ring" pattern that FA2 + circular init produces on sparse graphs.
|
||||
// Layout: community-seeded FA2 for dense graphs, random+FA2 for sparse.
|
||||
if (graph.order > 1) {
|
||||
// Detect communities — Louvain returns { nodeId: communityId }
|
||||
const communities = louvain(graph, { resolution: 1 });
|
||||
const edgeDensity = graph.size / graph.order; // edges per node
|
||||
|
||||
// Group nodes by community
|
||||
const communityGroups = new Map<number, string[]>();
|
||||
for (const node of graph.nodes()) {
|
||||
const c = communities[node] ?? 0;
|
||||
if (!communityGroups.has(c)) communityGroups.set(c, []);
|
||||
communityGroups.get(c)!.push(node);
|
||||
if (edgeDensity >= 0.8) {
|
||||
// Dense graph: Louvain community detection → grid seed → FA2 refinement.
|
||||
const communities = louvain(graph, { resolution: 1 });
|
||||
const communityGroups = new Map<number, string[]>();
|
||||
for (const node of graph.nodes()) {
|
||||
const c = communities[node] ?? 0;
|
||||
if (!communityGroups.has(c)) communityGroups.set(c, []);
|
||||
communityGroups.get(c)!.push(node);
|
||||
}
|
||||
const communityIds = Array.from(communityGroups.keys())
|
||||
.sort((a, b) => communityGroups.get(b)!.length - communityGroups.get(a)!.length);
|
||||
const numCommunities = communityIds.length;
|
||||
const maxCommunitySize = Math.max(
|
||||
...Array.from(communityGroups.values(), (nodes) => nodes.length),
|
||||
);
|
||||
const cellSize = Math.max(Math.sqrt(maxCommunitySize) * 28, 140);
|
||||
const cols = Math.ceil(Math.sqrt(numCommunities * 1.4));
|
||||
const gridWidth = cols * cellSize;
|
||||
const gridHeight = Math.ceil(numCommunities / cols) * cellSize;
|
||||
const jitter = (seed: number) => {
|
||||
const x = Math.sin(seed * 9999) * 10000;
|
||||
return (x - Math.floor(x)) - 0.5;
|
||||
};
|
||||
communityIds.forEach((cId, idx) => {
|
||||
const nodes = communityGroups.get(cId)!;
|
||||
const col = idx % cols;
|
||||
const row = Math.floor(idx / cols);
|
||||
const cx = col * cellSize - gridWidth / 2 + cellSize / 2 + jitter(idx) * cellSize * 0.2;
|
||||
const cy = row * cellSize - gridHeight / 2 + cellSize / 2 + jitter(idx + 1000) * cellSize * 0.2;
|
||||
const localRadius = Math.max(Math.sqrt(nodes.length) * 12, 25);
|
||||
nodes.forEach((nodeId, i) => {
|
||||
const angle = (i / nodes.length) * Math.PI * 2;
|
||||
const r = localRadius * (0.6 + Math.abs(jitter(i + idx * 100)) * 0.7);
|
||||
graph.setNodeAttribute(nodeId, "x", cx + Math.cos(angle) * r);
|
||||
graph.setNodeAttribute(nodeId, "y", cy + Math.sin(angle) * r);
|
||||
});
|
||||
});
|
||||
} else {
|
||||
// Sparse graph: random disc init with uniform area distribution.
|
||||
const spread = Math.sqrt(graph.order) * 20;
|
||||
const nodes = graph.nodes();
|
||||
for (let i = 0; i < nodes.length; i++) {
|
||||
const angle = Math.random() * Math.PI * 2;
|
||||
const r = Math.sqrt(Math.random()) * spread;
|
||||
graph.setNodeAttribute(nodes[i], "x", Math.cos(angle) * r);
|
||||
graph.setNodeAttribute(nodes[i], "y", Math.sin(angle) * r);
|
||||
}
|
||||
}
|
||||
|
||||
// Sort communities by size (big ones placed first / more central)
|
||||
const communityIds = Array.from(communityGroups.keys())
|
||||
.sort((a, b) => communityGroups.get(b)!.length - communityGroups.get(a)!.length);
|
||||
const numCommunities = communityIds.length;
|
||||
|
||||
// Cell size derived from the LARGEST community — smaller clusters get same cell
|
||||
// but will pack tighter via noverlap post-processing.
|
||||
const maxCommunitySize = Math.max(
|
||||
...Array.from(communityGroups.values(), (nodes) => nodes.length),
|
||||
);
|
||||
// Each node contributes ~12 units of space (node size ~5-12 + margin)
|
||||
const cellSize = Math.max(Math.sqrt(maxCommunitySize) * 28, 140);
|
||||
|
||||
// Grid dimensions — aim for slightly wide aspect (4:3)
|
||||
const cols = Math.ceil(Math.sqrt(numCommunities * 1.4));
|
||||
const rows = Math.ceil(numCommunities / cols);
|
||||
const gridWidth = cols * cellSize;
|
||||
const gridHeight = rows * cellSize;
|
||||
|
||||
// Deterministic pseudo-random for jitter (breaks grid rigidity)
|
||||
const jitter = (seed: number) => {
|
||||
const x = Math.sin(seed * 9999) * 10000;
|
||||
return (x - Math.floor(x)) - 0.5;
|
||||
};
|
||||
|
||||
communityIds.forEach((cId, idx) => {
|
||||
const nodesInCommunity = communityGroups.get(cId)!;
|
||||
const col = idx % cols;
|
||||
const row = Math.floor(idx / cols);
|
||||
// Cell center with slight jitter
|
||||
const cx = col * cellSize - gridWidth / 2 + cellSize / 2 + jitter(idx) * cellSize * 0.2;
|
||||
const cy = row * cellSize - gridHeight / 2 + cellSize / 2 + jitter(idx + 1000) * cellSize * 0.2;
|
||||
// Local radius: scales with community size — bigger community = bigger circle
|
||||
// Uses sqrt so large clusters don't balloon out of proportion
|
||||
const localRadius = Math.max(Math.sqrt(nodesInCommunity.length) * 12, 25);
|
||||
|
||||
nodesInCommunity.forEach((nodeId, i) => {
|
||||
const localAngle = (i / nodesInCommunity.length) * Math.PI * 2;
|
||||
const jitterR = localRadius * (0.6 + Math.abs(jitter(i + idx * 100)) * 0.7);
|
||||
const x = cx + Math.cos(localAngle) * jitterR;
|
||||
const y = cy + Math.sin(localAngle) * jitterR;
|
||||
graph.setNodeAttribute(nodeId, "x", x);
|
||||
graph.setNodeAttribute(nodeId, "y", y);
|
||||
});
|
||||
});
|
||||
|
||||
// Step 2: Run FA2 with community-aware settings.
|
||||
// Since we have community seeds, FA2 just refines (doesn't completely re-layout).
|
||||
const iterations = graph.order < 100 ? 250 : graph.order < 500 ? 180 : 100;
|
||||
// FA2 layout — Gephi-like defaults tuned per density.
|
||||
// Sparse: gravity ≈ repulsion so orphans form loose cloud (not ring/grid).
|
||||
// Dense: lower gravity, community seeds provide structure.
|
||||
const isSparse = edgeDensity < 0.8;
|
||||
const iterations = graph.order < 100 ? 300 : graph.order < 500 ? 200 : 120;
|
||||
forceAtlas2.assign(graph, {
|
||||
iterations,
|
||||
settings: {
|
||||
// Standard FA2 (NOT linLogMode — causes ring collapse)
|
||||
linLogMode: false,
|
||||
outboundAttractionDistribution: false,
|
||||
// Moderate gravity — pulls clusters slightly closer (prevents drift)
|
||||
gravity: 0.15,
|
||||
// Moderate repulsion — less aggressive spacing
|
||||
scalingRatio: 8,
|
||||
adjustSizes: true,
|
||||
gravity: isSparse ? 3.0 : 0.15,
|
||||
scalingRatio: isSparse ? 5.0 : 8,
|
||||
adjustSizes: false,
|
||||
strongGravityMode: false,
|
||||
slowDown: 8,
|
||||
slowDown: 6,
|
||||
barnesHutOptimize: graph.order > 300,
|
||||
edgeWeightInfluence: 0,
|
||||
},
|
||||
});
|
||||
|
||||
// Step 3: Noverlap post-process — push apart any remaining overlaps (tight)
|
||||
noverlap.assign(graph, {
|
||||
maxIterations: 50,
|
||||
settings: {
|
||||
margin: 3,
|
||||
ratio: 1.02,
|
||||
speed: 3,
|
||||
gridSize: 20,
|
||||
},
|
||||
});
|
||||
// Noverlap only for dense graphs — sparse graphs rely on FA2 repulsion
|
||||
// to space nodes naturally. Noverlap on sparse data creates grid artifacts.
|
||||
if (!isSparse) {
|
||||
noverlap.assign(graph, {
|
||||
maxIterations: 50,
|
||||
settings: { margin: 3, ratio: 1.02, speed: 3, gridSize: 20 },
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
const edgePrograms: Record<string, typeof EdgeArrowProgram> = {
|
||||
|
||||
@@ -92,38 +92,14 @@ export function useVaultSearchAll() {
|
||||
return { search };
|
||||
}
|
||||
|
||||
/** Fetch all links for a set of vault documents (for graph view). */
|
||||
/** Fetch all links for a set of vault documents (single batch query). */
|
||||
export function useVaultAllLinks(agentId: string, documents: { id: string }[]) {
|
||||
const http = useHttp();
|
||||
|
||||
const docIds = useMemo(() => documents.map((d) => d.id), [documents]);
|
||||
|
||||
const { data, isLoading } = useQuery({
|
||||
queryKey: [VAULT_KEY, "all-links", agentId, [...docIds].sort().join(",")],
|
||||
queryFn: async () => {
|
||||
if (docIds.length === 0) return [];
|
||||
const allLinks: VaultLink[] = [];
|
||||
const batchSize = 10;
|
||||
for (let i = 0; i < docIds.length; i += batchSize) {
|
||||
const batch = docIds.slice(i, i + batchSize);
|
||||
const results = await Promise.all(
|
||||
batch.map((id) =>
|
||||
http.get<{ outlinks: VaultLink[]; backlinks: VaultLink[] }>(
|
||||
`/v1/vault/documents/${id}/links`,
|
||||
).catch(() => ({ outlinks: [], backlinks: [] })),
|
||||
),
|
||||
);
|
||||
for (const r of results) {
|
||||
allLinks.push(...r.outlinks);
|
||||
}
|
||||
}
|
||||
const seen = new Set<string>();
|
||||
return allLinks.filter((l) => {
|
||||
if (seen.has(l.id)) return false;
|
||||
seen.add(l.id);
|
||||
return true;
|
||||
});
|
||||
},
|
||||
queryFn: () => http.post<VaultLink[]>("/v1/vault/links/batch", { doc_ids: docIds }),
|
||||
enabled: docIds.length > 0,
|
||||
staleTime: 60_000,
|
||||
});
|
||||
@@ -157,32 +133,11 @@ export function useVaultGraphData(agentId: string, opts?: { teamId?: string }) {
|
||||
[documents],
|
||||
);
|
||||
|
||||
// Fetch links for all docs via cross-agent endpoint (works for null agent_id too).
|
||||
// Fetch all links in one batch query (single SQL, no N+1).
|
||||
const docIds = useMemo(() => documents.map((d) => d.id), [documents]);
|
||||
const { data: linksData, isLoading: linksLoading } = useQuery({
|
||||
queryKey: [VAULT_KEY, "graph-links", docIdKey],
|
||||
queryFn: async () => {
|
||||
if (documents.length === 0) return [];
|
||||
const allLinks: VaultLink[] = [];
|
||||
const batchSize = 10;
|
||||
for (let i = 0; i < documents.length; i += batchSize) {
|
||||
const batch = documents.slice(i, i + batchSize);
|
||||
const results = await Promise.all(
|
||||
batch.map((doc) =>
|
||||
http.get<{ outlinks: VaultLink[]; backlinks: VaultLink[] }>(
|
||||
`/v1/vault/documents/${doc.id}/links`,
|
||||
).catch(() => ({ outlinks: [], backlinks: [] })),
|
||||
),
|
||||
);
|
||||
for (const r of results) allLinks.push(...r.outlinks);
|
||||
}
|
||||
// Dedup
|
||||
const seen = new Set<string>();
|
||||
return allLinks.filter((l) => {
|
||||
if (seen.has(l.id)) return false;
|
||||
seen.add(l.id);
|
||||
return true;
|
||||
});
|
||||
},
|
||||
queryFn: () => http.post<VaultLink[]>("/v1/vault/links/batch", { doc_ids: docIds }),
|
||||
enabled: documents.length > 0,
|
||||
staleTime: 60_000,
|
||||
});
|
||||
|
||||
@@ -278,7 +278,7 @@ export function VaultDetailDialog({ doc, open, onOpenChange, onDeleted }: Props)
|
||||
|
||||
{linkDialogOpen && (
|
||||
<VaultLinkDialog
|
||||
agentId={doc.agent_id}
|
||||
agentId={doc.agent_id ?? ""}
|
||||
fromDoc={doc}
|
||||
open={linkDialogOpen}
|
||||
onOpenChange={setLinkDialogOpen}
|
||||
|
||||
@@ -172,7 +172,7 @@ export function VaultDetailPanel({ doc, open, onClose, onDeleted }: Props) {
|
||||
</div>
|
||||
|
||||
{linkDialogOpen && (
|
||||
<VaultLinkDialog agentId={doc.agent_id} fromDoc={doc}
|
||||
<VaultLinkDialog agentId={doc.agent_id ?? ""} fromDoc={doc}
|
||||
open={linkDialogOpen} onOpenChange={setLinkDialogOpen} />
|
||||
)}
|
||||
</>
|
||||
|
||||
@@ -37,7 +37,12 @@ export function VaultGraphView({ agentId, teamId, selectedDocId, onNodeSelect, o
|
||||
const isLimited = totalCount > nodeLimit;
|
||||
const documents = useMemo(() => limitVaultDocsByDegree(allDocs, links, nodeLimit), [allDocs, links, nodeLimit]);
|
||||
const docMap = useMemo(() => new Map(documents.map((d) => [d.id, d])), [documents]);
|
||||
const graph = useMemo(() => buildVaultGraph(documents, links), [documents, links]);
|
||||
// Only build graph when ALL data loaded — prevents double-render
|
||||
// (orphan-only layout → with-links layout) that causes visual chaos.
|
||||
const graph = useMemo(
|
||||
() => loading ? buildVaultGraph([], []) : buildVaultGraph(documents, links),
|
||||
[documents, links, loading],
|
||||
);
|
||||
|
||||
const handleNodeDoubleClick = useCallback((nodeId: string) => {
|
||||
const doc = docMap.get(nodeId);
|
||||
|
||||
Reference in new issue
Block a user