mirror of
https://github.com/tiennm99/goclaw.git
synced 2026-10-12 12:13:51 +00:00
Chandan Kumar
65dbbcfd59
vault_search only indexed title + path + the auto-summary, and the summary is written from the first 3000 runes of the file. Anything the summary left out (room names, prices, codes) could never be found. On top of that the FTS query used plainto_tsquery, which ANDs every word, so a normal question like "what is the price of the Deluxe Ocean Suite?" matched nothing even when the key words were indexed. - Add vault_document_chunks (migration 98): the file body split into chunks with their own tsvector and embedding. body_indexed_hash on vault_documents records which content_hash the chunks came from, so unchanged files are not re-chunked or re-embedded. - The enrich worker rebuilds chunks when a file changes. This needs no LLM, so it runs even when no provider is configured. - Rescan backfills chunks for docs indexed before this change, since they already have a summary and never go back through the worker. - FTS matches any query word; ts_rank still ranks docs with more matching words first. Both FTS and vector search look at the doc and its chunks and score each doc by its best hit. SQLite is unchanged: its vault search is LIKE on title/path only.
30 lines
1.5 KiB
SQL
30 lines
1.5 KiB
SQL
-- Index vault document bodies in chunks (#1554). vault_documents.tsv and
|
|
-- embedding only cover title + path + summary, so terms that the auto-summary
|
|
-- drops (item names, prices, codes) could never be found by vault_search.
|
|
--
|
|
-- Chunks are derived data: the enrich worker rebuilds them from the workspace
|
|
-- file. body_indexed_hash records the content_hash the chunks were built from,
|
|
-- so unchanged files are not re-chunked or re-embedded, and rows with a NULL
|
|
-- or stale hash are picked up again by POST /v1/vault/rescan.
|
|
CREATE TABLE IF NOT EXISTS vault_document_chunks (
|
|
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
|
|
tenant_id UUID NOT NULL REFERENCES tenants(id) ON DELETE CASCADE,
|
|
document_id UUID NOT NULL REFERENCES vault_documents(id) ON DELETE CASCADE,
|
|
chunk_index INT NOT NULL,
|
|
start_line INT NOT NULL DEFAULT 0,
|
|
end_line INT NOT NULL DEFAULT 0,
|
|
text TEXT NOT NULL,
|
|
embedding vector(1536),
|
|
-- 'simple' config (no stemming), same as vault_documents.tsv
|
|
tsv tsvector GENERATED ALWAYS AS (to_tsvector('simple', text)) STORED,
|
|
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
|
UNIQUE (document_id, chunk_index)
|
|
);
|
|
|
|
CREATE INDEX IF NOT EXISTS idx_vault_chunks_tenant ON vault_document_chunks(tenant_id);
|
|
CREATE INDEX IF NOT EXISTS idx_vault_chunks_tsv ON vault_document_chunks USING GIN(tsv);
|
|
CREATE INDEX IF NOT EXISTS idx_vault_chunks_embedding ON vault_document_chunks
|
|
USING hnsw(embedding vector_cosine_ops);
|
|
|
|
ALTER TABLE vault_documents ADD COLUMN IF NOT EXISTS body_indexed_hash TEXT;
|