diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 01752509..68075831 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -18,22 +18,24 @@ jobs: contents: read packages: write steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + with: + persist-credentials: false - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 + uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3.12.0 with: driver: docker-container install: true - name: Login to DockerHub - uses: docker/login-action@v3 + uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3.7.0 with: username: ${{ secrets.DOCKER_USERNAME }} password: ${{ secrets.DOCKER_PASSWORD }} - name: Login to ghcr.io - uses: docker/login-action@v3 + uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3.7.0 with: registry: ghcr.io username: ${{ github.repository_owner }} @@ -41,7 +43,7 @@ jobs: - name: Image metadata (OCI labels) id: meta - uses: docker/metadata-action@v5 + uses: docker/metadata-action@c299e40c65443455700f0fdfc63efafe5b349051 # v5.10.0 with: images: | ${{ secrets.DOCKER_USERNAME }}/docsgpt @@ -51,7 +53,7 @@ jobs: org.opencontainers.image.version=${{ github.event.release.tag_name }} - name: Build and push platform-specific images - uses: docker/build-push-action@v6 + uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6.19.2 with: file: './application/Dockerfile' platforms: ${{ matrix.platform }} @@ -80,19 +82,19 @@ jobs: packages: write steps: - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 + uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3.12.0 with: driver: docker-container install: true - name: Login to DockerHub - uses: docker/login-action@v3 + uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3.7.0 with: username: ${{ secrets.DOCKER_USERNAME }} password: ${{ secrets.DOCKER_PASSWORD }} - name: Login to ghcr.io - uses: docker/login-action@v3 + uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3.7.0 with: registry: ghcr.io username: ${{ github.repository_owner }} @@ -120,11 +122,13 @@ jobs: permissions: contents: write steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + with: + persist-credentials: false - name: Attach the standalone compose file to the release env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + TAG: ${{ github.event.release.tag_name }} run: | - gh release upload "${{ github.event.release.tag_name }}" \ - deployment/docker-compose-standalone.yaml --clobber + gh release upload "$TAG" deployment/docker-compose-standalone.yaml --clobber diff --git a/.github/workflows/docker-develop-build.yml b/.github/workflows/docker-develop-build.yml index 0f97a331..baf0ffef 100644 --- a/.github/workflows/docker-develop-build.yml +++ b/.github/workflows/docker-develop-build.yml @@ -20,22 +20,24 @@ jobs: contents: read packages: write steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + with: + persist-credentials: false - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 + uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3.12.0 with: driver: docker-container install: true - name: Login to DockerHub - uses: docker/login-action@v3 + uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3.7.0 with: username: ${{ secrets.DOCKER_USERNAME }} password: ${{ secrets.DOCKER_PASSWORD }} - name: Login to ghcr.io - uses: docker/login-action@v3 + uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3.7.0 with: registry: ghcr.io username: ${{ github.repository_owner }} @@ -43,7 +45,7 @@ jobs: - name: Image metadata (OCI labels) id: meta - uses: docker/metadata-action@v5 + uses: docker/metadata-action@c299e40c65443455700f0fdfc63efafe5b349051 # v5.10.0 with: images: | ${{ secrets.DOCKER_USERNAME }}/docsgpt @@ -53,7 +55,7 @@ jobs: org.opencontainers.image.version=develop - name: Build and push platform-specific images - uses: docker/build-push-action@v6 + uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6.19.2 with: file: './application/Dockerfile' platforms: ${{ matrix.platform }} @@ -82,19 +84,19 @@ jobs: packages: write steps: - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 + uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3.12.0 with: driver: docker-container install: true - name: Login to DockerHub - uses: docker/login-action@v3 + uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3.7.0 with: username: ${{ secrets.DOCKER_USERNAME }} password: ${{ secrets.DOCKER_PASSWORD }} - name: Login to ghcr.io - uses: docker/login-action@v3 + uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3.7.0 with: registry: ghcr.io username: ${{ github.repository_owner }} diff --git a/.github/workflows/docker-image-verify.yml b/.github/workflows/docker-image-verify.yml index 4ac525c2..878c0cdd 100644 --- a/.github/workflows/docker-image-verify.yml +++ b/.github/workflows/docker-image-verify.yml @@ -23,30 +23,44 @@ permissions: jobs: verify: + strategy: + matrix: + # "" is the slim default; "-docling" bakes docling, its models and + # tesseract in, so the conversion check in verify_offline runs too. + variant: ["", "-docling"] runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + with: + persist-credentials: false - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 + uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3.12.0 - - name: Build the slim image - uses: docker/build-push-action@v6 + - name: Build the image + uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6.19.2 with: file: ./application/Dockerfile context: ./application platforms: linux/amd64 load: true - tags: docsgpt:verify - cache-from: type=gha - cache-to: type=gha,mode=max + tags: docsgpt:verify${{ matrix.variant }} + build-args: | + EXTRAS=${{ matrix.variant == '-docling' && 'docling' || '' }} + INSTALL_TESSERACT=${{ matrix.variant == '-docling' && 'true' || 'false' }} + cache-from: type=gha,scope=verify${{ matrix.variant }} + cache-to: type=gha,mode=max,scope=verify${{ matrix.variant }} - name: Image size + env: + IMAGE: docsgpt:verify${{ matrix.variant }} run: | - docker image inspect docsgpt:verify --format '{{.Size}}' | awk '{printf "uncompressed: %.2f GB\n", $1/1e9}' - docker history docsgpt:verify --format '{{.Size}}\t{{.CreatedBy}}' | head -20 + docker image inspect "$IMAGE" --format '{{.Size}}' | awk '{printf "uncompressed: %.2f GB\n", $1/1e9}' + docker history "$IMAGE" --format '{{.Size}}\t{{.CreatedBy}}' | head -20 - name: Offline verification (no network) + env: + IMAGE: docsgpt:verify${{ matrix.variant }} run: | - docker run --rm --network none docsgpt:verify \ + docker run --rm --network none "$IMAGE" \ python -m application.scripts.verify_offline diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 8f731dc5..061b35a1 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -26,9 +26,11 @@ jobs: # change to pyproject.toml or uv.lock was not re-exported. runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + with: + persist-credentials: false - - uses: astral-sh/setup-uv@v6 + - uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6.8.0 - name: Re-export and diff run: | diff --git a/application/parser/file/docling_parser.py b/application/parser/file/docling_parser.py index cf771a73..883b7857 100644 --- a/application/parser/file/docling_parser.py +++ b/application/parser/file/docling_parser.py @@ -592,7 +592,13 @@ class DoclingParser(BaseParser): logger.info(f" force_full_page_ocr={self.force_full_page_ocr}") logger.info(f" ocr_engine={self.ocr_engine or settings.OCR_ENGINE}") - if importlib.util.find_spec("docling.document_converter") is None: + # find_spec raises when the parent package is absent, so the hint has + # to cover both a missing docling and a docling without the submodule. + try: + converter_spec = importlib.util.find_spec("docling.document_converter") + except ModuleNotFoundError: + converter_spec = None + if converter_spec is None: raise ImportError(f"docling is required for DoclingParser. {install_hint('docling')}") # Create converter with hybrid OCR (smart: text direct, bitmaps OCR'd) diff --git a/deployment/docker-compose-azure.yaml b/deployment/docker-compose-azure.yaml index f276e6cc..ede47a0a 100644 --- a/deployment/docker-compose-azure.yaml +++ b/deployment/docker-compose-azure.yaml @@ -4,8 +4,20 @@ services: context: ../frontend target: dev environment: + # Every VITE_* the app reads, taken from .env (or the shell); empty ones + # leave the image's build-time default in place. - VITE_API_HOST=http://localhost:7091 - - VITE_API_STREAMING=$VITE_API_STREAMING + - VITE_API_STREAMING=${VITE_API_STREAMING:-true} + - VITE_BASE_URL=${VITE_BASE_URL:-} + - VITE_GOOGLE_CLIENT_ID=${VITE_GOOGLE_CLIENT_ID:-} + - VITE_GOOGLE_PICKER_API_KEY=${VITE_GOOGLE_PICKER_API_KEY:-} + - VITE_SHARE_POINT_CLIENT_ID=${VITE_SHARE_POINT_CLIENT_ID:-} + - VITE_CONFLUENCE_CLIENT_ID=${VITE_CONFLUENCE_CLIENT_ID:-} + - VITE_NOTIFICATION_TEXT=${VITE_NOTIFICATION_TEXT:-} + - VITE_NOTIFICATION_LINK=${VITE_NOTIFICATION_LINK:-} + - VITE_ENABLE_VOICE_INPUT=${VITE_ENABLE_VOICE_INPUT:-} + - VITE_DISABLE_SOURCE_FE=${VITE_DISABLE_SOURCE_FE:-} + - VITE_USE_V=${VITE_USE_V:-} ports: - "5173:5173" depends_on: diff --git a/deployment/docker-compose-hub.yaml b/deployment/docker-compose-hub.yaml index 40f7533c..cd365cca 100644 --- a/deployment/docker-compose-hub.yaml +++ b/deployment/docker-compose-hub.yaml @@ -10,9 +10,20 @@ services: frontend: image: arc53/docsgpt-fe:${DOCSGPT_IMAGE_TAG:-develop} environment: + # Every VITE_* the app reads, taken from .env (or the shell); empty ones + # leave the image's build-time default in place. - VITE_API_HOST=http://localhost:7091 - VITE_API_STREAMING=${VITE_API_STREAMING:-true} + - VITE_BASE_URL=${VITE_BASE_URL:-} - VITE_GOOGLE_CLIENT_ID=${VITE_GOOGLE_CLIENT_ID:-} + - VITE_GOOGLE_PICKER_API_KEY=${VITE_GOOGLE_PICKER_API_KEY:-} + - VITE_SHARE_POINT_CLIENT_ID=${VITE_SHARE_POINT_CLIENT_ID:-} + - VITE_CONFLUENCE_CLIENT_ID=${VITE_CONFLUENCE_CLIENT_ID:-} + - VITE_NOTIFICATION_TEXT=${VITE_NOTIFICATION_TEXT:-} + - VITE_NOTIFICATION_LINK=${VITE_NOTIFICATION_LINK:-} + - VITE_ENABLE_VOICE_INPUT=${VITE_ENABLE_VOICE_INPUT:-} + - VITE_DISABLE_SOURCE_FE=${VITE_DISABLE_SOURCE_FE:-} + - VITE_USE_V=${VITE_USE_V:-} ports: - "5173:5173" depends_on: diff --git a/deployment/docker-compose-standalone.yaml b/deployment/docker-compose-standalone.yaml index e1512459..8b09ea2c 100644 --- a/deployment/docker-compose-standalone.yaml +++ b/deployment/docker-compose-standalone.yaml @@ -29,8 +29,20 @@ services: - path: .env required: false environment: + # Every VITE_* the app reads, taken from .env (or the shell); empty ones + # leave the image's build-time default in place. - VITE_API_HOST=${VITE_API_HOST:-http://localhost:7091} - VITE_API_STREAMING=${VITE_API_STREAMING:-true} + - VITE_BASE_URL=${VITE_BASE_URL:-} + - VITE_GOOGLE_CLIENT_ID=${VITE_GOOGLE_CLIENT_ID:-} + - VITE_GOOGLE_PICKER_API_KEY=${VITE_GOOGLE_PICKER_API_KEY:-} + - VITE_SHARE_POINT_CLIENT_ID=${VITE_SHARE_POINT_CLIENT_ID:-} + - VITE_CONFLUENCE_CLIENT_ID=${VITE_CONFLUENCE_CLIENT_ID:-} + - VITE_NOTIFICATION_TEXT=${VITE_NOTIFICATION_TEXT:-} + - VITE_NOTIFICATION_LINK=${VITE_NOTIFICATION_LINK:-} + - VITE_ENABLE_VOICE_INPUT=${VITE_ENABLE_VOICE_INPUT:-} + - VITE_DISABLE_SOURCE_FE=${VITE_DISABLE_SOURCE_FE:-} + - VITE_USE_V=${VITE_USE_V:-} ports: - "5173:5173" depends_on: diff --git a/deployment/docker-compose.yaml b/deployment/docker-compose.yaml index fe9e9fd4..887e2a13 100644 --- a/deployment/docker-compose.yaml +++ b/deployment/docker-compose.yaml @@ -9,9 +9,20 @@ services: volumes: - ../frontend/src:/app/src environment: + # Every VITE_* the app reads, taken from .env (or the shell); empty ones + # leave the image's build-time default in place. - VITE_API_HOST=http://localhost:7091 - - VITE_API_STREAMING=$VITE_API_STREAMING - - VITE_GOOGLE_CLIENT_ID=$VITE_GOOGLE_CLIENT_ID + - VITE_API_STREAMING=${VITE_API_STREAMING:-true} + - VITE_BASE_URL=${VITE_BASE_URL:-} + - VITE_GOOGLE_CLIENT_ID=${VITE_GOOGLE_CLIENT_ID:-} + - VITE_GOOGLE_PICKER_API_KEY=${VITE_GOOGLE_PICKER_API_KEY:-} + - VITE_SHARE_POINT_CLIENT_ID=${VITE_SHARE_POINT_CLIENT_ID:-} + - VITE_CONFLUENCE_CLIENT_ID=${VITE_CONFLUENCE_CLIENT_ID:-} + - VITE_NOTIFICATION_TEXT=${VITE_NOTIFICATION_TEXT:-} + - VITE_NOTIFICATION_LINK=${VITE_NOTIFICATION_LINK:-} + - VITE_ENABLE_VOICE_INPUT=${VITE_ENABLE_VOICE_INPUT:-} + - VITE_DISABLE_SOURCE_FE=${VITE_DISABLE_SOURCE_FE:-} + - VITE_USE_V=${VITE_USE_V:-} ports: - "5173:5173" depends_on: diff --git a/docs/content/Guides/ocr.mdx b/docs/content/Guides/ocr.mdx index 5c26de75..28c10c79 100644 --- a/docs/content/Guides/ocr.mdx +++ b/docs/content/Guides/ocr.mdx @@ -117,8 +117,8 @@ shell) bakes docling into locally built backend and worker images; `setup.sh` offers it as a follow-up to the OCR question. Compose reads build arguments from the shell or from the `.env` you pass with `--env-file .env` (not from the containers' `env_file`), so build with `docker compose --env-file .env -f deployment/docker-compose.yaml build` -as `setup.sh` does. Pre-built Docker Hub images (`docker-compose-hub.yaml`) -never include docling; install it in a derived image instead. Either way no +as `setup.sh` does. Of the pre-built Docker Hub images only the slim default +excludes docling; the `-docling` variant ships it with its models. Either way no code changes are needed — docling is picked up as the fallback engine (and, under `OCR_BACKEND=auto`, as the OCR backend) as soon as it is importable, and `DOC_PARSER_ENGINE=docling` makes it the diff --git a/frontend/.dockerignore b/frontend/.dockerignore index 50fd39ab..04bfd71a 100644 --- a/frontend/.dockerignore +++ b/frontend/.dockerignore @@ -1,7 +1,8 @@ node_modules/ dist/ -.env -.env.* +# Local overrides never belong in the image; .env.development and .env.production do. +.env.local +.env.*.local Dockerfile .dockerignore *.log diff --git a/frontend/Dockerfile b/frontend/Dockerfile index 007795a6..9a4abc3d 100644 --- a/frontend/Dockerfile +++ b/frontend/Dockerfile @@ -31,9 +31,12 @@ CMD ["npm", "run", "dev", "--", "--host"] FROM deps AS build COPY . . -# Bake nothing deployment-specific; the runtime script supplies it. -ENV VITE_API_HOST="" -RUN npm run build && \ +# The image used to run the dev server, so .env.development was its set of +# defaults (notification banner, Google client id, local API host). Keep them +# as the production build's baseline; the container's VITE_* values override +# any of them at start-up. +RUN cp .env.development .env.production.local && \ + npm run build && \ sed -i 's|||' dist/index.html diff --git a/frontend/docker/40-runtime-env.sh b/frontend/docker/40-runtime-env.sh index a99fee3a..44462bd8 100644 --- a/frontend/docker/40-runtime-env.sh +++ b/frontend/docker/40-runtime-env.sh @@ -11,7 +11,9 @@ out=/usr/share/nginx/html/config.js { printf 'window.__DOCSGPT_ENV__ = {' first=1 - env | grep -E '^VITE_[A-Za-z0-9_]+=' | while IFS='=' read -r key value; do + env | grep -E '^VITE_[A-Za-z0-9_]+=.' | while IFS='=' read -r key value; do + # Empty values are skipped above (=.) so a compose passthrough like + # ${VITE_X:-} leaves the build-time default in place. # JSON-escape backslashes and double quotes; values are plain URLs/ids. escaped=$(printf '%s' "$value" | sed 's/\\/\\\\/g; s/"/\\"/g') if [ "$first" -eq 1 ]; then first=0; else printf ','; fi diff --git a/frontend/src/conversation/ConversationBubble.tsx b/frontend/src/conversation/ConversationBubble.tsx index 97464e4b..15bdbbb0 100644 --- a/frontend/src/conversation/ConversationBubble.tsx +++ b/frontend/src/conversation/ConversationBubble.tsx @@ -38,7 +38,7 @@ import ResearchProgress from './ResearchProgress'; import { ToolCallsType } from './types'; import { wikiWriteActionKey, wikiWritePath } from './wikiToolCall'; -const DisableSourceFE = envVar('VITE_DISABLE_SOURCE_FE') || false; +const DisableSourceFE = envVar('VITE_DISABLE_SOURCE_FE') === 'true'; const ConversationBubble = forwardRef< HTMLDivElement, diff --git a/tests/parser/file/test_docling_parser.py b/tests/parser/file/test_docling_parser.py index 682f5435..a2e81680 100644 --- a/tests/parser/file/test_docling_parser.py +++ b/tests/parser/file/test_docling_parser.py @@ -68,6 +68,19 @@ class TestDoclingParserInitParser: with pytest.raises(ImportError, match="docling is required"): parser._init_parser() + def test_init_parser_names_the_extra_when_docling_is_absent(self, monkeypatch): + """A missing parent package makes find_spec raise; the hint must still show.""" + import sys + + from application.parser.file.docling_parser import DoclingParser + + for name in [m for m in sys.modules if m == "docling" or m.startswith("docling.")]: + monkeypatch.delitem(sys.modules, name) + monkeypatch.setitem(sys.modules, "docling", None) + + with pytest.raises(ImportError, match="requirements-docling.txt"): + DoclingParser()._init_parser() + def test_init_parser_success(self): from application.parser.file.docling_parser import DoclingParser diff --git a/tests/scripts/test_verify_offline.py b/tests/scripts/test_verify_offline.py new file mode 100644 index 00000000..a729dbf9 --- /dev/null +++ b/tests/scripts/test_verify_offline.py @@ -0,0 +1,51 @@ +"""Offline verification: every check must pass, docling only when installed.""" + +import sys +import types +from unittest.mock import patch + +from application.scripts import verify_offline + + +def _fake_tiktoken(monkeypatch): + module = types.ModuleType("tiktoken") + module.get_encoding = lambda name: types.SimpleNamespace(encode=lambda text: [1, 2]) + monkeypatch.setitem(sys.modules, "tiktoken", module) + + +class TestVerify: + def test_passes_when_every_check_passes(self, monkeypatch, capsys): + _fake_tiktoken(monkeypatch) + counter = types.SimpleNamespace(name="org/model", count=lambda text: 4) + with patch("application.parser.tokenization.get_token_counter", return_value=counter), \ + patch("application.vectorstore.embeddings_local.EmbeddingsWrapper") as wrapper, \ + patch.object(verify_offline, "is_available", return_value=False): + wrapper.return_value.embed_query.return_value = [0.0] * 768 + assert verify_offline.verify(["ibm-granite/granite-embedding-311m-multilingual-r2"]) is True + out = capsys.readouterr().out + assert "ok tiktoken cl100k_base" in out + assert "skip docling" in out + + def test_fails_when_the_tokenizer_fell_back_to_cl100k(self, monkeypatch, capsys): + """A cache miss makes chunking silently use cl100k; that is a failed check.""" + _fake_tiktoken(monkeypatch) + counter = types.SimpleNamespace(name="cl100k_base", count=lambda text: 4) + with patch("application.parser.tokenization.get_token_counter", return_value=counter), \ + patch("application.vectorstore.embeddings_local.EmbeddingsWrapper") as wrapper, \ + patch.object(verify_offline, "is_available", return_value=False): + wrapper.return_value.embed_query.return_value = [0.0] * 768 + assert verify_offline.verify(["ibm-granite/granite-embedding-311m-multilingual-r2"]) is False + assert "FAIL tokenizer" in capsys.readouterr().out + + def test_runs_the_docling_check_when_installed(self, monkeypatch): + _fake_tiktoken(monkeypatch) + with patch.object(verify_offline, "is_available", return_value=True), \ + patch.object(verify_offline, "_docling_check", return_value="models from /app/models/docling") as check: + assert verify_offline.verify([]) is True + check.assert_called_once() + + def test_remote_models_are_skipped(self, monkeypatch, capsys): + _fake_tiktoken(monkeypatch) + with patch.object(verify_offline, "is_available", return_value=False): + assert verify_offline.verify(["openai_text-embedding-ada-002"]) is True + assert "skip openai_text-embedding-ada-002" in capsys.readouterr().out