feat: air-gapped deployment guide, no implicit downloads

- Ship tiktoken's cl100k_base inside the package and build the encoding
  from it, so token counting never downloads anything.
- Default EMBEDDINGS_CACHE_DIR to <data home>/models instead of FastEmbed's
  temp dir, and read tokenizer.json and repo metadata from that cache, so
  a model downloads once and survives reboots.
- TTS_PROVIDER=none and STT_PROVIDER=none switch the speech features off:
  the endpoints return 404, audio files fail to ingest with a clear
  message, /api/config reports tts_available/stt_available, and the UI
  hides the Speak and microphone buttons.
- Drop the Google Fonts Roboto import from the web UI.
- prefetch-models fills the cache the app reads; verify-offline checks the
  packaged encoding.
- Docs: new Air-Gapped Deployment guide, settings and cache notes.

docsgpt/utils.py

@@ -6,7 +6,7 @@ import logging
 import os
 import re
 import uuid
-from pathlib import PurePosixPath
+from pathlib import Path, PurePosixPath
 from typing import List
 
 import tiktoken
@@ -22,11 +22,44 @@ logger = logging.getLogger(__name__)
 
 _encoding = None
 
+# cl100k_base ships with the package; tiktoken would download it on first use.
+_CL100K_BASE_FILE = Path(__file__).resolve().parent / "core" / "encodings" / "cl100k_base.tiktoken"
+_CL100K_BASE_SHA256 = "223921b76ee99bde995b7ff738513eef100fb51d18c93597a113bcffe865b2a7"
+# Pattern and special tokens exactly as tiktoken_ext.openai_public defines cl100k_base.
+_CL100K_BASE_PAT_STR = (
+    r"""'(?i:[sdmt]|ll|ve|re)|[^\r\n\p{L}\p{N}]?+\p{L}++|\p{N}{1,3}+| ?[^\s\p{L}\p{N}]++[\r\n]*+|\s++$|\s*[\r\n]|\s+(?!\S)|\s"""
+)
+_CL100K_BASE_SPECIAL_TOKENS = {
+    "<|endoftext|>": 100257,
+    "<|fim_prefix|>": 100258,
+    "<|fim_middle|>": 100259,
+    "<|fim_suffix|>": 100260,
+    "<|endofprompt|>": 100276,
+}
 
-def get_encoding():
+
+def _load_cl100k_base() -> tiktoken.Encoding:
+    """Build cl100k_base from the packaged ranks file."""
+    data = _CL100K_BASE_FILE.read_bytes()
+    if hashlib.sha256(data).hexdigest() != _CL100K_BASE_SHA256:
+        raise ValueError(f"{_CL100K_BASE_FILE} does not match the cl100k_base checksum")
+    ranks = {
+        base64.b64decode(token): int(rank)
+        for token, rank in (line.split() for line in data.splitlines() if line)
+    }
+    return tiktoken.Encoding(
+        name="cl100k_base",
+        pat_str=_CL100K_BASE_PAT_STR,
+        mergeable_ranks=ranks,
+        special_tokens=_CL100K_BASE_SPECIAL_TOKENS,
+    )
+
+
+def get_encoding() -> tiktoken.Encoding:
+    """The cl100k_base encoding, loaded once per process."""
     global _encoding
     if _encoding is None:
-        _encoding = tiktoken.get_encoding("cl100k_base")
+        _encoding = _load_cl100k_base()
     return _encoding