mirror of
https://github.com/tiennm99/DocsGPT.git
synced 2026-10-11 03:12:55 +00:00
fix: address review on the slim-image branch
- The frontend image ran the Vite dev server in development mode, so .env.development supplied its defaults (notification banner, Google client id, local API host). The static build only loads .env.production, so the build stage now copies .env.development in as the baseline and the compose files pass every VITE_* the app reads through from .env; the runtime script skips empty values so a blank passthrough keeps the build-time default. .dockerignore kept only the .local variants out. - VITE_DISABLE_SOURCE_FE disables sources only when it is the string true. - DoclingParser: find_spec raises when docling itself is absent; the install hint now covers that path, with a regression test. - verify_offline: direct tests for verify(); the PR image check builds and verifies the -docling variant as well as slim. - Workflows this branch adds or rewrites pin actions by commit, pass the release tag through env instead of template expansion, and do not persist checkout credentials. - OCR guide no longer claims pre-built images never include docling.
This commit is contained in:
1 parent
588f110693
commit
6860a21541
16 files changed
+190
-46
No files matched your search
+16
-12
@@ -18,22 +18,24 @@ jobs:
|
||||
contents: read
|
||||
packages: write
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v3
|
||||
uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3.12.0
|
||||
with:
|
||||
driver: docker-container
|
||||
install: true
|
||||
|
||||
- name: Login to DockerHub
|
||||
uses: docker/login-action@v3
|
||||
uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3.7.0
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||
|
||||
- name: Login to ghcr.io
|
||||
uses: docker/login-action@v3
|
||||
uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3.7.0
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ github.repository_owner }}
|
||||
@@ -41,7 +43,7 @@ jobs:
|
||||
|
||||
- name: Image metadata (OCI labels)
|
||||
id: meta
|
||||
uses: docker/metadata-action@v5
|
||||
uses: docker/metadata-action@c299e40c65443455700f0fdfc63efafe5b349051 # v5.10.0
|
||||
with:
|
||||
images: |
|
||||
${{ secrets.DOCKER_USERNAME }}/docsgpt
|
||||
@@ -51,7 +53,7 @@ jobs:
|
||||
org.opencontainers.image.version=${{ github.event.release.tag_name }}
|
||||
|
||||
- name: Build and push platform-specific images
|
||||
uses: docker/build-push-action@v6
|
||||
uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6.19.2
|
||||
with:
|
||||
file: './application/Dockerfile'
|
||||
platforms: ${{ matrix.platform }}
|
||||
@@ -80,19 +82,19 @@ jobs:
|
||||
packages: write
|
||||
steps:
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v3
|
||||
uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3.12.0
|
||||
with:
|
||||
driver: docker-container
|
||||
install: true
|
||||
|
||||
- name: Login to DockerHub
|
||||
uses: docker/login-action@v3
|
||||
uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3.7.0
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||
|
||||
- name: Login to ghcr.io
|
||||
uses: docker/login-action@v3
|
||||
uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3.7.0
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ github.repository_owner }}
|
||||
@@ -120,11 +122,13 @@ jobs:
|
||||
permissions:
|
||||
contents: write
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Attach the standalone compose file to the release
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
TAG: ${{ github.event.release.tag_name }}
|
||||
run: |
|
||||
gh release upload "${{ github.event.release.tag_name }}" \
|
||||
deployment/docker-compose-standalone.yaml --clobber
|
||||
gh release upload "$TAG" deployment/docker-compose-standalone.yaml --clobber
|
||||
@@ -20,22 +20,24 @@ jobs:
|
||||
contents: read
|
||||
packages: write
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v3
|
||||
uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3.12.0
|
||||
with:
|
||||
driver: docker-container
|
||||
install: true
|
||||
|
||||
- name: Login to DockerHub
|
||||
uses: docker/login-action@v3
|
||||
uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3.7.0
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||
|
||||
- name: Login to ghcr.io
|
||||
uses: docker/login-action@v3
|
||||
uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3.7.0
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ github.repository_owner }}
|
||||
@@ -43,7 +45,7 @@ jobs:
|
||||
|
||||
- name: Image metadata (OCI labels)
|
||||
id: meta
|
||||
uses: docker/metadata-action@v5
|
||||
uses: docker/metadata-action@c299e40c65443455700f0fdfc63efafe5b349051 # v5.10.0
|
||||
with:
|
||||
images: |
|
||||
${{ secrets.DOCKER_USERNAME }}/docsgpt
|
||||
@@ -53,7 +55,7 @@ jobs:
|
||||
org.opencontainers.image.version=develop
|
||||
|
||||
- name: Build and push platform-specific images
|
||||
uses: docker/build-push-action@v6
|
||||
uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6.19.2
|
||||
with:
|
||||
file: './application/Dockerfile'
|
||||
platforms: ${{ matrix.platform }}
|
||||
@@ -82,19 +84,19 @@ jobs:
|
||||
packages: write
|
||||
steps:
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v3
|
||||
uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3.12.0
|
||||
with:
|
||||
driver: docker-container
|
||||
install: true
|
||||
|
||||
- name: Login to DockerHub
|
||||
uses: docker/login-action@v3
|
||||
uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3.7.0
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||
|
||||
- name: Login to ghcr.io
|
||||
uses: docker/login-action@v3
|
||||
uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3.7.0
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ github.repository_owner }}
|
||||
|
||||
@@ -23,30 +23,44 @@ permissions:
|
||||
|
||||
jobs:
|
||||
verify:
|
||||
strategy:
|
||||
matrix:
|
||||
# "" is the slim default; "-docling" bakes docling, its models and
|
||||
# tesseract in, so the conversion check in verify_offline runs too.
|
||||
variant: ["", "-docling"]
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v3
|
||||
uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3.12.0
|
||||
|
||||
- name: Build the slim image
|
||||
uses: docker/build-push-action@v6
|
||||
- name: Build the image
|
||||
uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6.19.2
|
||||
with:
|
||||
file: ./application/Dockerfile
|
||||
context: ./application
|
||||
platforms: linux/amd64
|
||||
load: true
|
||||
tags: docsgpt:verify
|
||||
cache-from: type=gha
|
||||
cache-to: type=gha,mode=max
|
||||
tags: docsgpt:verify${{ matrix.variant }}
|
||||
build-args: |
|
||||
EXTRAS=${{ matrix.variant == '-docling' && 'docling' || '' }}
|
||||
INSTALL_TESSERACT=${{ matrix.variant == '-docling' && 'true' || 'false' }}
|
||||
cache-from: type=gha,scope=verify${{ matrix.variant }}
|
||||
cache-to: type=gha,mode=max,scope=verify${{ matrix.variant }}
|
||||
|
||||
- name: Image size
|
||||
env:
|
||||
IMAGE: docsgpt:verify${{ matrix.variant }}
|
||||
run: |
|
||||
docker image inspect docsgpt:verify --format '{{.Size}}' | awk '{printf "uncompressed: %.2f GB\n", $1/1e9}'
|
||||
docker history docsgpt:verify --format '{{.Size}}\t{{.CreatedBy}}' | head -20
|
||||
docker image inspect "$IMAGE" --format '{{.Size}}' | awk '{printf "uncompressed: %.2f GB\n", $1/1e9}'
|
||||
docker history "$IMAGE" --format '{{.Size}}\t{{.CreatedBy}}' | head -20
|
||||
|
||||
- name: Offline verification (no network)
|
||||
env:
|
||||
IMAGE: docsgpt:verify${{ matrix.variant }}
|
||||
run: |
|
||||
docker run --rm --network none docsgpt:verify \
|
||||
docker run --rm --network none "$IMAGE" \
|
||||
python -m application.scripts.verify_offline
|
||||
@@ -26,9 +26,11 @@ jobs:
|
||||
# change to pyproject.toml or uv.lock was not re-exported.
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- uses: astral-sh/setup-uv@v6
|
||||
- uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6.8.0
|
||||
|
||||
- name: Re-export and diff
|
||||
run: |
|
||||
|
||||
@@ -592,7 +592,13 @@ class DoclingParser(BaseParser):
|
||||
logger.info(f" force_full_page_ocr={self.force_full_page_ocr}")
|
||||
logger.info(f" ocr_engine={self.ocr_engine or settings.OCR_ENGINE}")
|
||||
|
||||
if importlib.util.find_spec("docling.document_converter") is None:
|
||||
# find_spec raises when the parent package is absent, so the hint has
|
||||
# to cover both a missing docling and a docling without the submodule.
|
||||
try:
|
||||
converter_spec = importlib.util.find_spec("docling.document_converter")
|
||||
except ModuleNotFoundError:
|
||||
converter_spec = None
|
||||
if converter_spec is None:
|
||||
raise ImportError(f"docling is required for DoclingParser. {install_hint('docling')}")
|
||||
|
||||
# Create converter with hybrid OCR (smart: text direct, bitmaps OCR'd)
|
||||
|
||||
@@ -4,8 +4,20 @@ services:
|
||||
context: ../frontend
|
||||
target: dev
|
||||
environment:
|
||||
# Every VITE_* the app reads, taken from .env (or the shell); empty ones
|
||||
# leave the image's build-time default in place.
|
||||
- VITE_API_HOST=http://localhost:7091
|
||||
- VITE_API_STREAMING=$VITE_API_STREAMING
|
||||
- VITE_API_STREAMING=${VITE_API_STREAMING:-true}
|
||||
- VITE_BASE_URL=${VITE_BASE_URL:-}
|
||||
- VITE_GOOGLE_CLIENT_ID=${VITE_GOOGLE_CLIENT_ID:-}
|
||||
- VITE_GOOGLE_PICKER_API_KEY=${VITE_GOOGLE_PICKER_API_KEY:-}
|
||||
- VITE_SHARE_POINT_CLIENT_ID=${VITE_SHARE_POINT_CLIENT_ID:-}
|
||||
- VITE_CONFLUENCE_CLIENT_ID=${VITE_CONFLUENCE_CLIENT_ID:-}
|
||||
- VITE_NOTIFICATION_TEXT=${VITE_NOTIFICATION_TEXT:-}
|
||||
- VITE_NOTIFICATION_LINK=${VITE_NOTIFICATION_LINK:-}
|
||||
- VITE_ENABLE_VOICE_INPUT=${VITE_ENABLE_VOICE_INPUT:-}
|
||||
- VITE_DISABLE_SOURCE_FE=${VITE_DISABLE_SOURCE_FE:-}
|
||||
- VITE_USE_V=${VITE_USE_V:-}
|
||||
ports:
|
||||
- "5173:5173"
|
||||
depends_on:
|
||||
|
||||
@@ -10,9 +10,20 @@ services:
|
||||
frontend:
|
||||
image: arc53/docsgpt-fe:${DOCSGPT_IMAGE_TAG:-develop}
|
||||
environment:
|
||||
# Every VITE_* the app reads, taken from .env (or the shell); empty ones
|
||||
# leave the image's build-time default in place.
|
||||
- VITE_API_HOST=http://localhost:7091
|
||||
- VITE_API_STREAMING=${VITE_API_STREAMING:-true}
|
||||
- VITE_BASE_URL=${VITE_BASE_URL:-}
|
||||
- VITE_GOOGLE_CLIENT_ID=${VITE_GOOGLE_CLIENT_ID:-}
|
||||
- VITE_GOOGLE_PICKER_API_KEY=${VITE_GOOGLE_PICKER_API_KEY:-}
|
||||
- VITE_SHARE_POINT_CLIENT_ID=${VITE_SHARE_POINT_CLIENT_ID:-}
|
||||
- VITE_CONFLUENCE_CLIENT_ID=${VITE_CONFLUENCE_CLIENT_ID:-}
|
||||
- VITE_NOTIFICATION_TEXT=${VITE_NOTIFICATION_TEXT:-}
|
||||
- VITE_NOTIFICATION_LINK=${VITE_NOTIFICATION_LINK:-}
|
||||
- VITE_ENABLE_VOICE_INPUT=${VITE_ENABLE_VOICE_INPUT:-}
|
||||
- VITE_DISABLE_SOURCE_FE=${VITE_DISABLE_SOURCE_FE:-}
|
||||
- VITE_USE_V=${VITE_USE_V:-}
|
||||
ports:
|
||||
- "5173:5173"
|
||||
depends_on:
|
||||
|
||||
@@ -29,8 +29,20 @@ services:
|
||||
- path: .env
|
||||
required: false
|
||||
environment:
|
||||
# Every VITE_* the app reads, taken from .env (or the shell); empty ones
|
||||
# leave the image's build-time default in place.
|
||||
- VITE_API_HOST=${VITE_API_HOST:-http://localhost:7091}
|
||||
- VITE_API_STREAMING=${VITE_API_STREAMING:-true}
|
||||
- VITE_BASE_URL=${VITE_BASE_URL:-}
|
||||
- VITE_GOOGLE_CLIENT_ID=${VITE_GOOGLE_CLIENT_ID:-}
|
||||
- VITE_GOOGLE_PICKER_API_KEY=${VITE_GOOGLE_PICKER_API_KEY:-}
|
||||
- VITE_SHARE_POINT_CLIENT_ID=${VITE_SHARE_POINT_CLIENT_ID:-}
|
||||
- VITE_CONFLUENCE_CLIENT_ID=${VITE_CONFLUENCE_CLIENT_ID:-}
|
||||
- VITE_NOTIFICATION_TEXT=${VITE_NOTIFICATION_TEXT:-}
|
||||
- VITE_NOTIFICATION_LINK=${VITE_NOTIFICATION_LINK:-}
|
||||
- VITE_ENABLE_VOICE_INPUT=${VITE_ENABLE_VOICE_INPUT:-}
|
||||
- VITE_DISABLE_SOURCE_FE=${VITE_DISABLE_SOURCE_FE:-}
|
||||
- VITE_USE_V=${VITE_USE_V:-}
|
||||
ports:
|
||||
- "5173:5173"
|
||||
depends_on:
|
||||
|
||||
@@ -9,9 +9,20 @@ services:
|
||||
volumes:
|
||||
- ../frontend/src:/app/src
|
||||
environment:
|
||||
# Every VITE_* the app reads, taken from .env (or the shell); empty ones
|
||||
# leave the image's build-time default in place.
|
||||
- VITE_API_HOST=http://localhost:7091
|
||||
- VITE_API_STREAMING=$VITE_API_STREAMING
|
||||
- VITE_GOOGLE_CLIENT_ID=$VITE_GOOGLE_CLIENT_ID
|
||||
- VITE_API_STREAMING=${VITE_API_STREAMING:-true}
|
||||
- VITE_BASE_URL=${VITE_BASE_URL:-}
|
||||
- VITE_GOOGLE_CLIENT_ID=${VITE_GOOGLE_CLIENT_ID:-}
|
||||
- VITE_GOOGLE_PICKER_API_KEY=${VITE_GOOGLE_PICKER_API_KEY:-}
|
||||
- VITE_SHARE_POINT_CLIENT_ID=${VITE_SHARE_POINT_CLIENT_ID:-}
|
||||
- VITE_CONFLUENCE_CLIENT_ID=${VITE_CONFLUENCE_CLIENT_ID:-}
|
||||
- VITE_NOTIFICATION_TEXT=${VITE_NOTIFICATION_TEXT:-}
|
||||
- VITE_NOTIFICATION_LINK=${VITE_NOTIFICATION_LINK:-}
|
||||
- VITE_ENABLE_VOICE_INPUT=${VITE_ENABLE_VOICE_INPUT:-}
|
||||
- VITE_DISABLE_SOURCE_FE=${VITE_DISABLE_SOURCE_FE:-}
|
||||
- VITE_USE_V=${VITE_USE_V:-}
|
||||
ports:
|
||||
- "5173:5173"
|
||||
depends_on:
|
||||
|
||||
@@ -117,8 +117,8 @@ shell) bakes docling into locally built backend and worker images; `setup.sh`
|
||||
offers it as a follow-up to the OCR question. Compose reads build arguments from the shell or from the
|
||||
`.env` you pass with `--env-file .env` (not from the containers' `env_file`),
|
||||
so build with `docker compose --env-file .env -f deployment/docker-compose.yaml build`
|
||||
as `setup.sh` does. Pre-built Docker Hub images (`docker-compose-hub.yaml`)
|
||||
never include docling; install it in a derived image instead. Either way no
|
||||
as `setup.sh` does. Of the pre-built Docker Hub images only the slim default
|
||||
excludes docling; the `-docling` variant ships it with its models. Either way no
|
||||
code changes are needed — docling is picked up as
|
||||
the fallback engine (and, under `OCR_BACKEND=auto`, as the OCR backend) as
|
||||
soon as it is importable, and `DOC_PARSER_ENGINE=docling` makes it the
|
||||
|
||||
@@ -1,7 +1,8 @@
|
||||
node_modules/
|
||||
dist/
|
||||
.env
|
||||
.env.*
|
||||
# Local overrides never belong in the image; .env.development and .env.production do.
|
||||
.env.local
|
||||
.env.*.local
|
||||
Dockerfile
|
||||
.dockerignore
|
||||
*.log
|
||||
+6
-3
@@ -31,9 +31,12 @@ CMD ["npm", "run", "dev", "--", "--host"]
|
||||
FROM deps AS build
|
||||
|
||||
COPY . .
|
||||
# Bake nothing deployment-specific; the runtime script supplies it.
|
||||
ENV VITE_API_HOST=""
|
||||
RUN npm run build && \
|
||||
# The image used to run the dev server, so .env.development was its set of
|
||||
# defaults (notification banner, Google client id, local API host). Keep them
|
||||
# as the production build's baseline; the container's VITE_* values override
|
||||
# any of them at start-up.
|
||||
RUN cp .env.development .env.production.local && \
|
||||
npm run build && \
|
||||
sed -i 's|<head>|<head><script src="/config.js"></script>|' dist/index.html
|
||||
|
||||
|
||||
|
||||
@@ -11,7 +11,9 @@ out=/usr/share/nginx/html/config.js
|
||||
{
|
||||
printf 'window.__DOCSGPT_ENV__ = {'
|
||||
first=1
|
||||
env | grep -E '^VITE_[A-Za-z0-9_]+=' | while IFS='=' read -r key value; do
|
||||
env | grep -E '^VITE_[A-Za-z0-9_]+=.' | while IFS='=' read -r key value; do
|
||||
# Empty values are skipped above (=.) so a compose passthrough like
|
||||
# ${VITE_X:-} leaves the build-time default in place.
|
||||
# JSON-escape backslashes and double quotes; values are plain URLs/ids.
|
||||
escaped=$(printf '%s' "$value" | sed 's/\\/\\\\/g; s/"/\\"/g')
|
||||
if [ "$first" -eq 1 ]; then first=0; else printf ','; fi
|
||||
|
||||
@@ -38,7 +38,7 @@ import ResearchProgress from './ResearchProgress';
|
||||
import { ToolCallsType } from './types';
|
||||
import { wikiWriteActionKey, wikiWritePath } from './wikiToolCall';
|
||||
|
||||
const DisableSourceFE = envVar('VITE_DISABLE_SOURCE_FE') || false;
|
||||
const DisableSourceFE = envVar('VITE_DISABLE_SOURCE_FE') === 'true';
|
||||
|
||||
const ConversationBubble = forwardRef<
|
||||
HTMLDivElement,
|
||||
|
||||
@@ -68,6 +68,19 @@ class TestDoclingParserInitParser:
|
||||
with pytest.raises(ImportError, match="docling is required"):
|
||||
parser._init_parser()
|
||||
|
||||
def test_init_parser_names_the_extra_when_docling_is_absent(self, monkeypatch):
|
||||
"""A missing parent package makes find_spec raise; the hint must still show."""
|
||||
import sys
|
||||
|
||||
from application.parser.file.docling_parser import DoclingParser
|
||||
|
||||
for name in [m for m in sys.modules if m == "docling" or m.startswith("docling.")]:
|
||||
monkeypatch.delitem(sys.modules, name)
|
||||
monkeypatch.setitem(sys.modules, "docling", None)
|
||||
|
||||
with pytest.raises(ImportError, match="requirements-docling.txt"):
|
||||
DoclingParser()._init_parser()
|
||||
|
||||
def test_init_parser_success(self):
|
||||
from application.parser.file.docling_parser import DoclingParser
|
||||
|
||||
|
||||
@@ -0,0 +1,51 @@
|
||||
"""Offline verification: every check must pass, docling only when installed."""
|
||||
|
||||
import sys
|
||||
import types
|
||||
from unittest.mock import patch
|
||||
|
||||
from application.scripts import verify_offline
|
||||
|
||||
|
||||
def _fake_tiktoken(monkeypatch):
|
||||
module = types.ModuleType("tiktoken")
|
||||
module.get_encoding = lambda name: types.SimpleNamespace(encode=lambda text: [1, 2])
|
||||
monkeypatch.setitem(sys.modules, "tiktoken", module)
|
||||
|
||||
|
||||
class TestVerify:
|
||||
def test_passes_when_every_check_passes(self, monkeypatch, capsys):
|
||||
_fake_tiktoken(monkeypatch)
|
||||
counter = types.SimpleNamespace(name="org/model", count=lambda text: 4)
|
||||
with patch("application.parser.tokenization.get_token_counter", return_value=counter), \
|
||||
patch("application.vectorstore.embeddings_local.EmbeddingsWrapper") as wrapper, \
|
||||
patch.object(verify_offline, "is_available", return_value=False):
|
||||
wrapper.return_value.embed_query.return_value = [0.0] * 768
|
||||
assert verify_offline.verify(["ibm-granite/granite-embedding-311m-multilingual-r2"]) is True
|
||||
out = capsys.readouterr().out
|
||||
assert "ok tiktoken cl100k_base" in out
|
||||
assert "skip docling" in out
|
||||
|
||||
def test_fails_when_the_tokenizer_fell_back_to_cl100k(self, monkeypatch, capsys):
|
||||
"""A cache miss makes chunking silently use cl100k; that is a failed check."""
|
||||
_fake_tiktoken(monkeypatch)
|
||||
counter = types.SimpleNamespace(name="cl100k_base", count=lambda text: 4)
|
||||
with patch("application.parser.tokenization.get_token_counter", return_value=counter), \
|
||||
patch("application.vectorstore.embeddings_local.EmbeddingsWrapper") as wrapper, \
|
||||
patch.object(verify_offline, "is_available", return_value=False):
|
||||
wrapper.return_value.embed_query.return_value = [0.0] * 768
|
||||
assert verify_offline.verify(["ibm-granite/granite-embedding-311m-multilingual-r2"]) is False
|
||||
assert "FAIL tokenizer" in capsys.readouterr().out
|
||||
|
||||
def test_runs_the_docling_check_when_installed(self, monkeypatch):
|
||||
_fake_tiktoken(monkeypatch)
|
||||
with patch.object(verify_offline, "is_available", return_value=True), \
|
||||
patch.object(verify_offline, "_docling_check", return_value="models from /app/models/docling") as check:
|
||||
assert verify_offline.verify([]) is True
|
||||
check.assert_called_once()
|
||||
|
||||
def test_remote_models_are_skipped(self, monkeypatch, capsys):
|
||||
_fake_tiktoken(monkeypatch)
|
||||
with patch.object(verify_offline, "is_available", return_value=False):
|
||||
assert verify_offline.verify(["openai_text-embedding-ada-002"]) is True
|
||||
assert "skip openai_text-embedding-ada-002" in capsys.readouterr().out
|
||||
Reference in new issue
Block a user