mirror of
https://github.com/tiennm99/DocsGPT.git
synced 2026-10-15 12:13:38 +00:00
fix(parser,vectorstore): stop the first-request downloads in a warmed install
Three things still reached the network from a container whose models were baked in: - tiktoken fetched cl100k_base from openaipublic.blob.core.windows.net on every fresh container (its cache defaulted to /tmp), and token accounting calls it on every chat. prefetch_models now warms it too; the image sets TIKTOKEN_CACHE_DIR. - The chunker loaded its tokenizer with Tokenizer.from_pretrained, which revalidates the revision with a HEAD request per process start and stalls for the etag timeout (10 s) when huggingface.co is unreachable. It now reads tokenizer.json from the hub cache first and only downloads on a miss; the repo-metadata read for models outside the registry does the same. - tldextract fetched the public suffix list on the first web crawl; the bundled snapshot is used instead. application/scripts/verify_offline.py exercises these paths (and docling's conversion when the extra is installed) so an image can be checked with docker run --network none.
This commit is contained in:
1 parent
196865846c
commit
4707c45b93
8 files changed
+262
-10
No files matched your search
@@ -1,6 +1,9 @@
|
||||
"""Chunk sizes must be counted in the embedding model's units, and splitting
|
||||
must never rewrite the text it splits."""
|
||||
|
||||
import sys
|
||||
import types
|
||||
|
||||
import pytest
|
||||
|
||||
from application.parser import tokenization
|
||||
@@ -298,3 +301,35 @@ class TestUnknownTokenCollapse:
|
||||
assert "".join(pieces) == text, "split must not lose or alter text"
|
||||
assert len(pieces) > 1, "a collapsed run must still be cut into pieces"
|
||||
assert all(counter.count(p) <= 20 for p in pieces)
|
||||
|
||||
|
||||
class TestTokenizerFile:
|
||||
"""The chunker's tokenizer must come from the hub cache without a network round trip."""
|
||||
|
||||
def test_cache_hit_makes_no_online_call(self, monkeypatch):
|
||||
calls = []
|
||||
|
||||
def fake_download(repo, filename, local_files_only=False):
|
||||
calls.append(local_files_only)
|
||||
return "/cache/tokenizer.json"
|
||||
|
||||
fake_hub = types.ModuleType("huggingface_hub")
|
||||
fake_hub.hf_hub_download = fake_download
|
||||
monkeypatch.setitem(sys.modules, "huggingface_hub", fake_hub)
|
||||
assert tokenization._tokenizer_file("org/model") == "/cache/tokenizer.json"
|
||||
assert calls == [True]
|
||||
|
||||
def test_cache_miss_falls_back_to_online(self, monkeypatch):
|
||||
calls = []
|
||||
|
||||
def fake_download(repo, filename, local_files_only=False):
|
||||
calls.append(local_files_only)
|
||||
if local_files_only:
|
||||
raise FileNotFoundError("not cached")
|
||||
return "/downloaded/tokenizer.json"
|
||||
|
||||
fake_hub = types.ModuleType("huggingface_hub")
|
||||
fake_hub.hf_hub_download = fake_download
|
||||
monkeypatch.setitem(sys.modules, "huggingface_hub", fake_hub)
|
||||
assert tokenization._tokenizer_file("org/model") == "/downloaded/tokenizer.json"
|
||||
assert calls == [True, False]
|
||||
Reference in new issue
Block a user