From 6860a215417acce92c136d604f74c7b392f5d9c7 Mon Sep 17 00:00:00 2001 From: Alex Date: Sun, 6 Sep 2026 21:44:34 +0100 Subject: [PATCH] fix: address review on the slim-image branch - The frontend image ran the Vite dev server in development mode, so .env.development supplied its defaults (notification banner, Google client id, local API host). The static build only loads .env.production, so the build stage now copies .env.development in as the baseline and the compose files pass every VITE_* the app reads through from .env; the runtime script skips empty values so a blank passthrough keeps the build-time default. .dockerignore kept only the .local variants out. - VITE_DISABLE_SOURCE_FE disables sources only when it is the string true. - DoclingParser: find_spec raises when docling itself is absent; the install hint now covers that path, with a regression test. - verify_offline: direct tests for verify(); the PR image check builds and verifies the -docling variant as well as slim. - Workflows this branch adds or rewrites pin actions by commit, pass the release tag through env instead of template expansion, and do not persist checkout credentials. - OCR guide no longer claims pre-built images never include docling. --- .github/workflows/ci.yml | 28 +++++----- .github/workflows/docker-develop-build.yml | 20 ++++---- .github/workflows/docker-image-verify.yml | 34 +++++++++---- .github/workflows/lint.yml | 6 ++- application/parser/file/docling_parser.py | 8 ++- deployment/docker-compose-azure.yaml | 14 ++++- deployment/docker-compose-hub.yaml | 11 ++++ deployment/docker-compose-standalone.yaml | 12 +++++ deployment/docker-compose.yaml | 15 +++++- docs/content/Guides/ocr.mdx | 4 +- frontend/.dockerignore | 5 +- frontend/Dockerfile | 9 ++-- frontend/docker/40-runtime-env.sh | 4 +- .../src/conversation/ConversationBubble.tsx | 2 +- tests/parser/file/test_docling_parser.py | 13 +++++ tests/scripts/test_verify_offline.py | 51 +++++++++++++++++++ 16 files changed, 190 insertions(+), 46 deletions(-) create mode 100644 tests/scripts/test_verify_offline.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 01752509..68075831 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -18,22 +18,24 @@ jobs: contents: read packages: write steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + with: + persist-credentials: false - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 + uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3.12.0 with: driver: docker-container install: true - name: Login to DockerHub - uses: docker/login-action@v3 + uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3.7.0 with: username: ${{ secrets.DOCKER_USERNAME }} password: ${{ secrets.DOCKER_PASSWORD }} - name: Login to ghcr.io - uses: docker/login-action@v3 + uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3.7.0 with: registry: ghcr.io username: ${{ github.repository_owner }} @@ -41,7 +43,7 @@ jobs: - name: Image metadata (OCI labels) id: meta - uses: docker/metadata-action@v5 + uses: docker/metadata-action@c299e40c65443455700f0fdfc63efafe5b349051 # v5.10.0 with: images: | ${{ secrets.DOCKER_USERNAME }}/docsgpt @@ -51,7 +53,7 @@ jobs: org.opencontainers.image.version=${{ github.event.release.tag_name }} - name: Build and push platform-specific images - uses: docker/build-push-action@v6 + uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6.19.2 with: file: './application/Dockerfile' platforms: ${{ matrix.platform }} @@ -80,19 +82,19 @@ jobs: packages: write steps: - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 + uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3.12.0 with: driver: docker-container install: true - name: Login to DockerHub - uses: docker/login-action@v3 + uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3.7.0 with: username: ${{ secrets.DOCKER_USERNAME }} password: ${{ secrets.DOCKER_PASSWORD }} - name: Login to ghcr.io - uses: docker/login-action@v3 + uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3.7.0 with: registry: ghcr.io username: ${{ github.repository_owner }} @@ -120,11 +122,13 @@ jobs: permissions: contents: write steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + with: + persist-credentials: false - name: Attach the standalone compose file to the release env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + TAG: ${{ github.event.release.tag_name }} run: | - gh release upload "${{ github.event.release.tag_name }}" \ - deployment/docker-compose-standalone.yaml --clobber + gh release upload "$TAG" deployment/docker-compose-standalone.yaml --clobber diff --git a/.github/workflows/docker-develop-build.yml b/.github/workflows/docker-develop-build.yml index 0f97a331..baf0ffef 100644 --- a/.github/workflows/docker-develop-build.yml +++ b/.github/workflows/docker-develop-build.yml @@ -20,22 +20,24 @@ jobs: contents: read packages: write steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + with: + persist-credentials: false - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 + uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3.12.0 with: driver: docker-container install: true - name: Login to DockerHub - uses: docker/login-action@v3 + uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3.7.0 with: username: ${{ secrets.DOCKER_USERNAME }} password: ${{ secrets.DOCKER_PASSWORD }} - name: Login to ghcr.io - uses: docker/login-action@v3 + uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3.7.0 with: registry: ghcr.io username: ${{ github.repository_owner }} @@ -43,7 +45,7 @@ jobs: - name: Image metadata (OCI labels) id: meta - uses: docker/metadata-action@v5 + uses: docker/metadata-action@c299e40c65443455700f0fdfc63efafe5b349051 # v5.10.0 with: images: | ${{ secrets.DOCKER_USERNAME }}/docsgpt @@ -53,7 +55,7 @@ jobs: org.opencontainers.image.version=develop - name: Build and push platform-specific images - uses: docker/build-push-action@v6 + uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6.19.2 with: file: './application/Dockerfile' platforms: ${{ matrix.platform }} @@ -82,19 +84,19 @@ jobs: packages: write steps: - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 + uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3.12.0 with: driver: docker-container install: true - name: Login to DockerHub - uses: docker/login-action@v3 + uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3.7.0 with: username: ${{ secrets.DOCKER_USERNAME }} password: ${{ secrets.DOCKER_PASSWORD }} - name: Login to ghcr.io - uses: docker/login-action@v3 + uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3.7.0 with: registry: ghcr.io username: ${{ github.repository_owner }} diff --git a/.github/workflows/docker-image-verify.yml b/.github/workflows/docker-image-verify.yml index 4ac525c2..878c0cdd 100644 --- a/.github/workflows/docker-image-verify.yml +++ b/.github/workflows/docker-image-verify.yml @@ -23,30 +23,44 @@ permissions: jobs: verify: + strategy: + matrix: + # "" is the slim default; "-docling" bakes docling, its models and + # tesseract in, so the conversion check in verify_offline runs too. + variant: ["", "-docling"] runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + with: + persist-credentials: false - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 + uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3.12.0 - - name: Build the slim image - uses: docker/build-push-action@v6 + - name: Build the image + uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6.19.2 with: file: ./application/Dockerfile context: ./application platforms: linux/amd64 load: true - tags: docsgpt:verify - cache-from: type=gha - cache-to: type=gha,mode=max + tags: docsgpt:verify${{ matrix.variant }} + build-args: | + EXTRAS=${{ matrix.variant == '-docling' && 'docling' || '' }} + INSTALL_TESSERACT=${{ matrix.variant == '-docling' && 'true' || 'false' }} + cache-from: type=gha,scope=verify${{ matrix.variant }} + cache-to: type=gha,mode=max,scope=verify${{ matrix.variant }} - name: Image size + env: + IMAGE: docsgpt:verify${{ matrix.variant }} run: | - docker image inspect docsgpt:verify --format '{{.Size}}' | awk '{printf "uncompressed: %.2f GB\n", $1/1e9}' - docker history docsgpt:verify --format '{{.Size}}\t{{.CreatedBy}}' | head -20 + docker image inspect "$IMAGE" --format '{{.Size}}' | awk '{printf "uncompressed: %.2f GB\n", $1/1e9}' + docker history "$IMAGE" --format '{{.Size}}\t{{.CreatedBy}}' | head -20 - name: Offline verification (no network) + env: + IMAGE: docsgpt:verify${{ matrix.variant }} run: | - docker run --rm --network none docsgpt:verify \ + docker run --rm --network none "$IMAGE" \ python -m application.scripts.verify_offline diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 8f731dc5..061b35a1 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -26,9 +26,11 @@ jobs: # change to pyproject.toml or uv.lock was not re-exported. runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + with: + persist-credentials: false - - uses: astral-sh/setup-uv@v6 + - uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6.8.0 - name: Re-export and diff run: | diff --git a/application/parser/file/docling_parser.py b/application/parser/file/docling_parser.py index cf771a73..883b7857 100644 --- a/application/parser/file/docling_parser.py +++ b/application/parser/file/docling_parser.py @@ -592,7 +592,13 @@ class DoclingParser(BaseParser): logger.info(f" force_full_page_ocr={self.force_full_page_ocr}") logger.info(f" ocr_engine={self.ocr_engine or settings.OCR_ENGINE}") - if importlib.util.find_spec("docling.document_converter") is None: + # find_spec raises when the parent package is absent, so the hint has + # to cover both a missing docling and a docling without the submodule. + try: + converter_spec = importlib.util.find_spec("docling.document_converter") + except ModuleNotFoundError: + converter_spec = None + if converter_spec is None: raise ImportError(f"docling is required for DoclingParser. {install_hint('docling')}") # Create converter with hybrid OCR (smart: text direct, bitmaps OCR'd) diff --git a/deployment/docker-compose-azure.yaml b/deployment/docker-compose-azure.yaml index f276e6cc..ede47a0a 100644 --- a/deployment/docker-compose-azure.yaml +++ b/deployment/docker-compose-azure.yaml @@ -4,8 +4,20 @@ services: context: ../frontend target: dev environment: + # Every VITE_* the app reads, taken from .env (or the shell); empty ones + # leave the image's build-time default in place. - VITE_API_HOST=http://localhost:7091 - - VITE_API_STREAMING=$VITE_API_STREAMING + - VITE_API_STREAMING=${VITE_API_STREAMING:-true} + - VITE_BASE_URL=${VITE_BASE_URL:-} + - VITE_GOOGLE_CLIENT_ID=${VITE_GOOGLE_CLIENT_ID:-} + - VITE_GOOGLE_PICKER_API_KEY=${VITE_GOOGLE_PICKER_API_KEY:-} + - VITE_SHARE_POINT_CLIENT_ID=${VITE_SHARE_POINT_CLIENT_ID:-} + - VITE_CONFLUENCE_CLIENT_ID=${VITE_CONFLUENCE_CLIENT_ID:-} + - VITE_NOTIFICATION_TEXT=${VITE_NOTIFICATION_TEXT:-} + - VITE_NOTIFICATION_LINK=${VITE_NOTIFICATION_LINK:-} + - VITE_ENABLE_VOICE_INPUT=${VITE_ENABLE_VOICE_INPUT:-} + - VITE_DISABLE_SOURCE_FE=${VITE_DISABLE_SOURCE_FE:-} + - VITE_USE_V=${VITE_USE_V:-} ports: - "5173:5173" depends_on: diff --git a/deployment/docker-compose-hub.yaml b/deployment/docker-compose-hub.yaml index 40f7533c..cd365cca 100644 --- a/deployment/docker-compose-hub.yaml +++ b/deployment/docker-compose-hub.yaml @@ -10,9 +10,20 @@ services: frontend: image: arc53/docsgpt-fe:${DOCSGPT_IMAGE_TAG:-develop} environment: + # Every VITE_* the app reads, taken from .env (or the shell); empty ones + # leave the image's build-time default in place. - VITE_API_HOST=http://localhost:7091 - VITE_API_STREAMING=${VITE_API_STREAMING:-true} + - VITE_BASE_URL=${VITE_BASE_URL:-} - VITE_GOOGLE_CLIENT_ID=${VITE_GOOGLE_CLIENT_ID:-} + - VITE_GOOGLE_PICKER_API_KEY=${VITE_GOOGLE_PICKER_API_KEY:-} + - VITE_SHARE_POINT_CLIENT_ID=${VITE_SHARE_POINT_CLIENT_ID:-} + - VITE_CONFLUENCE_CLIENT_ID=${VITE_CONFLUENCE_CLIENT_ID:-} + - VITE_NOTIFICATION_TEXT=${VITE_NOTIFICATION_TEXT:-} + - VITE_NOTIFICATION_LINK=${VITE_NOTIFICATION_LINK:-} + - VITE_ENABLE_VOICE_INPUT=${VITE_ENABLE_VOICE_INPUT:-} + - VITE_DISABLE_SOURCE_FE=${VITE_DISABLE_SOURCE_FE:-} + - VITE_USE_V=${VITE_USE_V:-} ports: - "5173:5173" depends_on: diff --git a/deployment/docker-compose-standalone.yaml b/deployment/docker-compose-standalone.yaml index e1512459..8b09ea2c 100644 --- a/deployment/docker-compose-standalone.yaml +++ b/deployment/docker-compose-standalone.yaml @@ -29,8 +29,20 @@ services: - path: .env required: false environment: + # Every VITE_* the app reads, taken from .env (or the shell); empty ones + # leave the image's build-time default in place. - VITE_API_HOST=${VITE_API_HOST:-http://localhost:7091} - VITE_API_STREAMING=${VITE_API_STREAMING:-true} + - VITE_BASE_URL=${VITE_BASE_URL:-} + - VITE_GOOGLE_CLIENT_ID=${VITE_GOOGLE_CLIENT_ID:-} + - VITE_GOOGLE_PICKER_API_KEY=${VITE_GOOGLE_PICKER_API_KEY:-} + - VITE_SHARE_POINT_CLIENT_ID=${VITE_SHARE_POINT_CLIENT_ID:-} + - VITE_CONFLUENCE_CLIENT_ID=${VITE_CONFLUENCE_CLIENT_ID:-} + - VITE_NOTIFICATION_TEXT=${VITE_NOTIFICATION_TEXT:-} + - VITE_NOTIFICATION_LINK=${VITE_NOTIFICATION_LINK:-} + - VITE_ENABLE_VOICE_INPUT=${VITE_ENABLE_VOICE_INPUT:-} + - VITE_DISABLE_SOURCE_FE=${VITE_DISABLE_SOURCE_FE:-} + - VITE_USE_V=${VITE_USE_V:-} ports: - "5173:5173" depends_on: diff --git a/deployment/docker-compose.yaml b/deployment/docker-compose.yaml index fe9e9fd4..887e2a13 100644 --- a/deployment/docker-compose.yaml +++ b/deployment/docker-compose.yaml @@ -9,9 +9,20 @@ services: volumes: - ../frontend/src:/app/src environment: + # Every VITE_* the app reads, taken from .env (or the shell); empty ones + # leave the image's build-time default in place. - VITE_API_HOST=http://localhost:7091 - - VITE_API_STREAMING=$VITE_API_STREAMING - - VITE_GOOGLE_CLIENT_ID=$VITE_GOOGLE_CLIENT_ID + - VITE_API_STREAMING=${VITE_API_STREAMING:-true} + - VITE_BASE_URL=${VITE_BASE_URL:-} + - VITE_GOOGLE_CLIENT_ID=${VITE_GOOGLE_CLIENT_ID:-} + - VITE_GOOGLE_PICKER_API_KEY=${VITE_GOOGLE_PICKER_API_KEY:-} + - VITE_SHARE_POINT_CLIENT_ID=${VITE_SHARE_POINT_CLIENT_ID:-} + - VITE_CONFLUENCE_CLIENT_ID=${VITE_CONFLUENCE_CLIENT_ID:-} + - VITE_NOTIFICATION_TEXT=${VITE_NOTIFICATION_TEXT:-} + - VITE_NOTIFICATION_LINK=${VITE_NOTIFICATION_LINK:-} + - VITE_ENABLE_VOICE_INPUT=${VITE_ENABLE_VOICE_INPUT:-} + - VITE_DISABLE_SOURCE_FE=${VITE_DISABLE_SOURCE_FE:-} + - VITE_USE_V=${VITE_USE_V:-} ports: - "5173:5173" depends_on: diff --git a/docs/content/Guides/ocr.mdx b/docs/content/Guides/ocr.mdx index 5c26de75..28c10c79 100644 --- a/docs/content/Guides/ocr.mdx +++ b/docs/content/Guides/ocr.mdx @@ -117,8 +117,8 @@ shell) bakes docling into locally built backend and worker images; `setup.sh` offers it as a follow-up to the OCR question. Compose reads build arguments from the shell or from the `.env` you pass with `--env-file .env` (not from the containers' `env_file`), so build with `docker compose --env-file .env -f deployment/docker-compose.yaml build` -as `setup.sh` does. Pre-built Docker Hub images (`docker-compose-hub.yaml`) -never include docling; install it in a derived image instead. Either way no +as `setup.sh` does. Of the pre-built Docker Hub images only the slim default +excludes docling; the `-docling` variant ships it with its models. Either way no code changes are needed — docling is picked up as the fallback engine (and, under `OCR_BACKEND=auto`, as the OCR backend) as soon as it is importable, and `DOC_PARSER_ENGINE=docling` makes it the diff --git a/frontend/.dockerignore b/frontend/.dockerignore index 50fd39ab..04bfd71a 100644 --- a/frontend/.dockerignore +++ b/frontend/.dockerignore @@ -1,7 +1,8 @@ node_modules/ dist/ -.env -.env.* +# Local overrides never belong in the image; .env.development and .env.production do. +.env.local +.env.*.local Dockerfile .dockerignore *.log diff --git a/frontend/Dockerfile b/frontend/Dockerfile index 007795a6..9a4abc3d 100644 --- a/frontend/Dockerfile +++ b/frontend/Dockerfile @@ -31,9 +31,12 @@ CMD ["npm", "run", "dev", "--", "--host"] FROM deps AS build COPY . . -# Bake nothing deployment-specific; the runtime script supplies it. -ENV VITE_API_HOST="" -RUN npm run build && \ +# The image used to run the dev server, so .env.development was its set of +# defaults (notification banner, Google client id, local API host). Keep them +# as the production build's baseline; the container's VITE_* values override +# any of them at start-up. +RUN cp .env.development .env.production.local && \ + npm run build && \ sed -i 's|||' dist/index.html diff --git a/frontend/docker/40-runtime-env.sh b/frontend/docker/40-runtime-env.sh index a99fee3a..44462bd8 100644 --- a/frontend/docker/40-runtime-env.sh +++ b/frontend/docker/40-runtime-env.sh @@ -11,7 +11,9 @@ out=/usr/share/nginx/html/config.js { printf 'window.__DOCSGPT_ENV__ = {' first=1 - env | grep -E '^VITE_[A-Za-z0-9_]+=' | while IFS='=' read -r key value; do + env | grep -E '^VITE_[A-Za-z0-9_]+=.' | while IFS='=' read -r key value; do + # Empty values are skipped above (=.) so a compose passthrough like + # ${VITE_X:-} leaves the build-time default in place. # JSON-escape backslashes and double quotes; values are plain URLs/ids. escaped=$(printf '%s' "$value" | sed 's/\\/\\\\/g; s/"/\\"/g') if [ "$first" -eq 1 ]; then first=0; else printf ','; fi diff --git a/frontend/src/conversation/ConversationBubble.tsx b/frontend/src/conversation/ConversationBubble.tsx index 97464e4b..15bdbbb0 100644 --- a/frontend/src/conversation/ConversationBubble.tsx +++ b/frontend/src/conversation/ConversationBubble.tsx @@ -38,7 +38,7 @@ import ResearchProgress from './ResearchProgress'; import { ToolCallsType } from './types'; import { wikiWriteActionKey, wikiWritePath } from './wikiToolCall'; -const DisableSourceFE = envVar('VITE_DISABLE_SOURCE_FE') || false; +const DisableSourceFE = envVar('VITE_DISABLE_SOURCE_FE') === 'true'; const ConversationBubble = forwardRef< HTMLDivElement, diff --git a/tests/parser/file/test_docling_parser.py b/tests/parser/file/test_docling_parser.py index 682f5435..a2e81680 100644 --- a/tests/parser/file/test_docling_parser.py +++ b/tests/parser/file/test_docling_parser.py @@ -68,6 +68,19 @@ class TestDoclingParserInitParser: with pytest.raises(ImportError, match="docling is required"): parser._init_parser() + def test_init_parser_names_the_extra_when_docling_is_absent(self, monkeypatch): + """A missing parent package makes find_spec raise; the hint must still show.""" + import sys + + from application.parser.file.docling_parser import DoclingParser + + for name in [m for m in sys.modules if m == "docling" or m.startswith("docling.")]: + monkeypatch.delitem(sys.modules, name) + monkeypatch.setitem(sys.modules, "docling", None) + + with pytest.raises(ImportError, match="requirements-docling.txt"): + DoclingParser()._init_parser() + def test_init_parser_success(self): from application.parser.file.docling_parser import DoclingParser diff --git a/tests/scripts/test_verify_offline.py b/tests/scripts/test_verify_offline.py new file mode 100644 index 00000000..a729dbf9 --- /dev/null +++ b/tests/scripts/test_verify_offline.py @@ -0,0 +1,51 @@ +"""Offline verification: every check must pass, docling only when installed.""" + +import sys +import types +from unittest.mock import patch + +from application.scripts import verify_offline + + +def _fake_tiktoken(monkeypatch): + module = types.ModuleType("tiktoken") + module.get_encoding = lambda name: types.SimpleNamespace(encode=lambda text: [1, 2]) + monkeypatch.setitem(sys.modules, "tiktoken", module) + + +class TestVerify: + def test_passes_when_every_check_passes(self, monkeypatch, capsys): + _fake_tiktoken(monkeypatch) + counter = types.SimpleNamespace(name="org/model", count=lambda text: 4) + with patch("application.parser.tokenization.get_token_counter", return_value=counter), \ + patch("application.vectorstore.embeddings_local.EmbeddingsWrapper") as wrapper, \ + patch.object(verify_offline, "is_available", return_value=False): + wrapper.return_value.embed_query.return_value = [0.0] * 768 + assert verify_offline.verify(["ibm-granite/granite-embedding-311m-multilingual-r2"]) is True + out = capsys.readouterr().out + assert "ok tiktoken cl100k_base" in out + assert "skip docling" in out + + def test_fails_when_the_tokenizer_fell_back_to_cl100k(self, monkeypatch, capsys): + """A cache miss makes chunking silently use cl100k; that is a failed check.""" + _fake_tiktoken(monkeypatch) + counter = types.SimpleNamespace(name="cl100k_base", count=lambda text: 4) + with patch("application.parser.tokenization.get_token_counter", return_value=counter), \ + patch("application.vectorstore.embeddings_local.EmbeddingsWrapper") as wrapper, \ + patch.object(verify_offline, "is_available", return_value=False): + wrapper.return_value.embed_query.return_value = [0.0] * 768 + assert verify_offline.verify(["ibm-granite/granite-embedding-311m-multilingual-r2"]) is False + assert "FAIL tokenizer" in capsys.readouterr().out + + def test_runs_the_docling_check_when_installed(self, monkeypatch): + _fake_tiktoken(monkeypatch) + with patch.object(verify_offline, "is_available", return_value=True), \ + patch.object(verify_offline, "_docling_check", return_value="models from /app/models/docling") as check: + assert verify_offline.verify([]) is True + check.assert_called_once() + + def test_remote_models_are_skipped(self, monkeypatch, capsys): + _fake_tiktoken(monkeypatch) + with patch.object(verify_offline, "is_available", return_value=False): + assert verify_offline.verify(["openai_text-embedding-ada-002"]) is True + assert "skip openai_text-embedding-ada-002" in capsys.readouterr().out