mirror of
https://github.com/tiennm99/DocsGPT.git
synced 2026-10-18 10:14:24 +00:00
Keep the chunker's metadata when re-embedding a wiki page
`reembed_wiki_page_worker` built each chunk's metadata from scratch as
`{source, title, filename}`, discarding everything the chunker attached
to the document -- including the per-chunk `token_count` every other
ingest path records. Wiki chunks therefore reached the source viewer
with no count at all.
Carry `extra_info` over and let the page's own path and title win on top,
so a chunk that inherited a stale source from an earlier conversion is
still keyed and filtered correctly.
This commit is contained in:
1 parent
0aecdc32c9
commit
d787f54c2d
2 files changed
+80
-4
No files matched your search
+6
-4
@@ -2615,10 +2615,12 @@ def reembed_wiki_page_worker(self, source_id, path, content_hash, user):
|
||||
|
||||
added = 0
|
||||
for chunk in chunks:
|
||||
store.add_chunk(
|
||||
chunk.text,
|
||||
metadata={"source": path, "title": title, "filename": path},
|
||||
)
|
||||
# Start from what the chunker produced -- ``token_count`` above
|
||||
# all, which the source viewer reads per chunk -- and let the
|
||||
# page's own identity win over anything stale it inherited.
|
||||
metadata = dict(chunk.extra_info or {})
|
||||
metadata.update({"source": path, "title": title, "filename": path})
|
||||
store.add_chunk(chunk.text, metadata=metadata)
|
||||
added += 1
|
||||
|
||||
with db_session() as conn:
|
||||
|
||||
Reference in new issue
Block a user