diff --git a/.github/workflows/backend-release.yml b/.github/workflows/backend-release.yml index 4efe4665..2b766b01 100644 --- a/.github/workflows/backend-release.yml +++ b/.github/workflows/backend-release.yml @@ -1,7 +1,8 @@ name: Backend release # A version bump on main tags the commit, creates the GitHub release, then -# publishes the Docker images and the PyPI package by calling those workflows. +# publishes the backend, frontend and sandbox Docker images and the PyPI +# package by calling those workflows. # The release is created with GITHUB_TOKEN, and GitHub never starts workflows # from events that token produces, so the `release: published` triggers on the # publish workflows would not fire (0.18.0 got no images that way). Releases @@ -109,6 +110,32 @@ jobs: contents: write # release-assets attaches the compose file to the release packages: write + frontend: + needs: release + if: needs.release.outputs.created == 'true' + uses: $/.github/workflows/cife.yml + with: + version: ${{ needs.release.outputs.version }} + secrets: + DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }} + DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} + permissions: + contents: read + packages: write + + sandbox: + needs: release + if: needs.release.outputs.created == 'true' + uses: $/.github/workflows/sandbox-image.yml + with: + version: ${{ needs.release.outputs.version }} + secrets: + DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }} + DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} + permissions: + contents: read + packages: write + pypi: needs: release if: needs.release.outputs.created == 'true' diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index dc780607..b2985af5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -142,11 +142,15 @@ jobs: - name: Create and push multi-arch manifests env: TAG: ${{ env.RELEASE_TAG }}${{ matrix.variant }} - LATEST: latest${{ matrix.variant }} + # A stable release moves `latest`; a prerelease published by hand + # moves only its own tag (the release trigger fires for those too). + LATEST: ${{ (inputs.version || !github.event.release.prerelease) && format('latest{0}', matrix.variant) || '' }} run: | set -e + # $LATEST is deliberately unquoted: it is empty for a prerelease, and + # an empty word would create a manifest named "$repo:". for repo in "$DOCKERHUB_NAMESPACE/docsgpt" "ghcr.io/${{ github.repository_owner }}/docsgpt"; do - for name in "$TAG" "$LATEST"; do + for name in "$TAG" $LATEST; do docker manifest create "$repo:$name" \ --amend "$repo:$TAG-amd64" \ --amend "$repo:$TAG-arm64" diff --git a/.github/workflows/cife.yml b/.github/workflows/cife.yml index 1118723f..6c8c301f 100644 --- a/.github/workflows/cife.yml +++ b/.github/workflows/cife.yml @@ -1,12 +1,47 @@ name: Build and push DocsGPT-FE Docker image +# Runs for a release created by hand (the release event), or called by the +# backend-release workflow with the version it just tagged: GitHub never starts +# workflows from events GITHUB_TOKEN produces, so a bot-created release does not +# fire the release trigger on its own. + on: release: types: [published] + workflow_call: + inputs: + version: + description: Release tag to build and push (the images are tagged with it) + type: string + required: true + secrets: + DOCKER_USERNAME: + required: true + DOCKER_PASSWORD: + required: true + +permissions: + contents: read + +env: + # The tag being published: passed in by the caller, or the release's own. + RELEASE_TAG: ${{ inputs.version || github.event.release.tag_name }} + # A stable release also moves `latest`. A prerelease published by hand moves + # only its own tag: the release trigger fires for those too, and they must not + # become `latest`. + MOVING_TAG: ${{ (inputs.version || !github.event.release.prerelease) && 'latest' || '' }} jobs: build: if: github.repository == 'arc53/DocsGPT' + # Publishing jobs run in a GitHub Actions environment so the registry + # credentials can be scoped to it and protection rules (required reviewers, + # branch restrictions) applied in the repository settings. + environment: docker-hub + env: + # Public namespace the compose files pull from; the login secret only + # authenticates the push. + DOCKERHUB_NAMESPACE: arc53 strategy: matrix: include: @@ -21,92 +56,103 @@ jobs: contents: read packages: write steps: - - uses: actions/checkout@v4 - - - name: Set up QEMU # Only needed for emulation, not for native arm64 builds - if: matrix.platform == 'linux/arm64' - uses: docker/setup-qemu-action@v3 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + with: + persist-credentials: false - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 + uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3.12.0 with: driver: docker-container install: true - name: Login to DockerHub - uses: docker/login-action@v3 + uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3.7.0 with: username: ${{ secrets.DOCKER_USERNAME }} password: ${{ secrets.DOCKER_PASSWORD }} - name: Login to ghcr.io - uses: docker/login-action@v3 + uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3.7.0 with: registry: ghcr.io username: ${{ github.repository_owner }} password: ${{ secrets.GITHUB_TOKEN }} + - name: Image metadata (OCI labels) + id: meta + uses: docker/metadata-action@c299e40c65443455700f0fdfc63efafe5b349051 # v5.10.0 + with: + images: | + ${{ env.DOCKERHUB_NAMESPACE }}/docsgpt-fe + ghcr.io/${{ github.repository_owner }}/docsgpt-fe + labels: | + org.opencontainers.image.title=DocsGPT-FE + org.opencontainers.image.version=${{ env.RELEASE_TAG }} + - name: Build and push platform-specific images - uses: docker/build-push-action@v6 + uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6.19.2 with: file: './frontend/Dockerfile' platforms: ${{ matrix.platform }} context: ./frontend push: true tags: | - ${{ secrets.DOCKER_USERNAME }}/docsgpt-fe:${{ github.event.release.tag_name }}-${{ matrix.suffix }} - ghcr.io/${{ github.repository_owner }}/docsgpt-fe:${{ github.event.release.tag_name }}-${{ matrix.suffix }} + ${{ env.DOCKERHUB_NAMESPACE }}/docsgpt-fe:${{ env.RELEASE_TAG }}-${{ matrix.suffix }} + ghcr.io/${{ github.repository_owner }}/docsgpt-fe:${{ env.RELEASE_TAG }}-${{ matrix.suffix }} + labels: ${{ steps.meta.outputs.labels }} provenance: false sbom: false - cache-from: type=registry,ref=${{ secrets.DOCKER_USERNAME }}/docsgpt-fe:latest + cache-from: type=registry,ref=${{ env.DOCKERHUB_NAMESPACE }}/docsgpt-fe:latest cache-to: type=inline manifest: if: github.repository == 'arc53/DocsGPT' + # Publishing jobs run in a GitHub Actions environment so the registry + # credentials can be scoped to it and protection rules (required reviewers, + # branch restrictions) applied in the repository settings. + environment: docker-hub + env: + # Public namespace the compose files pull from; the login secret only + # authenticates the push. + DOCKERHUB_NAMESPACE: arc53 needs: build runs-on: ubuntu-latest permissions: packages: write steps: - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 + uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3.12.0 with: driver: docker-container install: true - name: Login to DockerHub - uses: docker/login-action@v3 + uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3.7.0 with: username: ${{ secrets.DOCKER_USERNAME }} password: ${{ secrets.DOCKER_PASSWORD }} - name: Login to ghcr.io - uses: docker/login-action@v3 + uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3.7.0 with: registry: ghcr.io username: ${{ github.repository_owner }} password: ${{ secrets.GITHUB_TOKEN }} - - name: Create and push manifest for DockerHub + - name: Create and push multi-arch manifests + env: + TAG: ${{ env.RELEASE_TAG }} + MOVING: ${{ env.MOVING_TAG }} run: | set -e - docker manifest create ${{ secrets.DOCKER_USERNAME }}/docsgpt-fe:${{ github.event.release.tag_name }} \ - --amend ${{ secrets.DOCKER_USERNAME }}/docsgpt-fe:${{ github.event.release.tag_name }}-amd64 \ - --amend ${{ secrets.DOCKER_USERNAME }}/docsgpt-fe:${{ github.event.release.tag_name }}-arm64 - docker manifest push ${{ secrets.DOCKER_USERNAME }}/docsgpt-fe:${{ github.event.release.tag_name }} - docker manifest create ${{ secrets.DOCKER_USERNAME }}/docsgpt-fe:latest \ - --amend ${{ secrets.DOCKER_USERNAME }}/docsgpt-fe:${{ github.event.release.tag_name }}-amd64 \ - --amend ${{ secrets.DOCKER_USERNAME }}/docsgpt-fe:${{ github.event.release.tag_name }}-arm64 - docker manifest push ${{ secrets.DOCKER_USERNAME }}/docsgpt-fe:latest - - - name: Create and push manifest for ghcr.io - run: | - set -e - docker manifest create ghcr.io/${{ github.repository_owner }}/docsgpt-fe:${{ github.event.release.tag_name }} \ - --amend ghcr.io/${{ github.repository_owner }}/docsgpt-fe:${{ github.event.release.tag_name }}-amd64 \ - --amend ghcr.io/${{ github.repository_owner }}/docsgpt-fe:${{ github.event.release.tag_name }}-arm64 - docker manifest push ghcr.io/${{ github.repository_owner }}/docsgpt-fe:${{ github.event.release.tag_name }} - docker manifest create ghcr.io/${{ github.repository_owner }}/docsgpt-fe:latest \ - --amend ghcr.io/${{ github.repository_owner }}/docsgpt-fe:${{ github.event.release.tag_name }}-amd64 \ - --amend ghcr.io/${{ github.repository_owner }}/docsgpt-fe:${{ github.event.release.tag_name }}-arm64 - docker manifest push ghcr.io/${{ github.repository_owner }}/docsgpt-fe:latest \ No newline at end of file + # $MOVING is deliberately unquoted: it is empty for a prerelease, and + # an empty word would create a manifest named "$repo:". + for repo in "$DOCKERHUB_NAMESPACE/docsgpt-fe" "ghcr.io/${{ github.repository_owner }}/docsgpt-fe"; do + for name in "$TAG" $MOVING; do + docker manifest create "$repo:$name" \ + --amend "$repo:$TAG-amd64" \ + --amend "$repo:$TAG-arm64" + docker manifest push "$repo:$name" + done + done diff --git a/.github/workflows/sandbox-image.yml b/.github/workflows/sandbox-image.yml new file mode 100644 index 00000000..c26e5b8d --- /dev/null +++ b/.github/workflows/sandbox-image.yml @@ -0,0 +1,170 @@ +name: Build and push the docsgpt-sandbox image + +# The opt-in code-execution runner (deployment/sandbox). Compose builds it from +# the checkout, but Kubernetes cannot build, so the manifest under +# deployment/k8s/deployments/sandbox-deploy.yaml needs a published image. +# +# Three ways in: a push to main that touches the runner (tagged `develop`), a +# release created by hand (the release event), or a call from the backend-release +# workflow with the version it just tagged — GitHub never starts workflows from +# events GITHUB_TOKEN produces, so a bot-created release does not fire the +# release trigger on its own. `github.event_name` is the event that started the +# whole run, which is `push` when backend-release calls this, so the tag comes +# from the inputs and the release payload instead. + +on: + release: + types: [published] + workflow_call: + inputs: + version: + description: Release tag to build and push (the images are tagged with it) + type: string + required: true + secrets: + DOCKER_USERNAME: + required: true + DOCKER_PASSWORD: + required: true + push: + branches: [main] + paths: + - 'deployment/sandbox/**' + - '.github/workflows/sandbox-image.yml' + +permissions: + contents: read + +env: + # The version being published, or `develop` for a push to main. + RELEASE_TAG: ${{ inputs.version || github.event.release.tag_name || 'develop' }} + # A stable release also moves `latest`. A push to main moves nothing but + # `develop`, and a prerelease published by hand moves only its own tag: the + # release trigger fires for those too, and they must not become `latest`. + MOVING_TAG: ${{ (inputs.version || (github.event.release.tag_name && !github.event.release.prerelease)) && 'latest' || '' }} + +jobs: + build: + if: github.repository == 'arc53/DocsGPT' + # Publishing jobs run in a GitHub Actions environment so the registry + # credentials can be scoped to it and protection rules (required reviewers, + # branch restrictions) applied in the repository settings. + environment: docker-hub + env: + # Public namespace the compose files and manifests pull from; the login + # secret only authenticates the push. + DOCKERHUB_NAMESPACE: arc53 + strategy: + matrix: + include: + - platform: linux/amd64 + runner: ubuntu-latest + suffix: amd64 + - platform: linux/arm64 + runner: ubuntu-24.04-arm + suffix: arm64 + runs-on: ${{ matrix.runner }} + permissions: + contents: read + packages: write + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + with: + persist-credentials: false + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3.12.0 + with: + driver: docker-container + install: true + + - name: Login to DockerHub + uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3.7.0 + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} + + - name: Login to ghcr.io + uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3.7.0 + with: + registry: ghcr.io + username: ${{ github.repository_owner }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Image metadata (OCI labels) + id: meta + uses: docker/metadata-action@c299e40c65443455700f0fdfc63efafe5b349051 # v5.10.0 + with: + images: | + ${{ env.DOCKERHUB_NAMESPACE }}/docsgpt-sandbox + ghcr.io/${{ github.repository_owner }}/docsgpt-sandbox + labels: | + org.opencontainers.image.title=DocsGPT sandbox runner + org.opencontainers.image.version=${{ env.RELEASE_TAG }} + + - name: Build and push platform-specific images + uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6.19.2 + with: + file: './deployment/sandbox/Dockerfile' + platforms: ${{ matrix.platform }} + context: ./deployment/sandbox + push: true + tags: | + ${{ env.DOCKERHUB_NAMESPACE }}/docsgpt-sandbox:${{ env.RELEASE_TAG }}-${{ matrix.suffix }} + ghcr.io/${{ github.repository_owner }}/docsgpt-sandbox:${{ env.RELEASE_TAG }}-${{ matrix.suffix }} + labels: ${{ steps.meta.outputs.labels }} + provenance: false + sbom: false + cache-from: type=registry,ref=${{ env.DOCKERHUB_NAMESPACE }}/docsgpt-sandbox:develop + cache-to: type=inline + + manifest: + if: github.repository == 'arc53/DocsGPT' + # Publishing jobs run in a GitHub Actions environment so the registry + # credentials can be scoped to it and protection rules (required reviewers, + # branch restrictions) applied in the repository settings. + environment: docker-hub + env: + # Public namespace the compose files and manifests pull from; the login + # secret only authenticates the push. + DOCKERHUB_NAMESPACE: arc53 + needs: build + runs-on: ubuntu-latest + permissions: + packages: write + steps: + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3.12.0 + with: + driver: docker-container + install: true + + - name: Login to DockerHub + uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3.7.0 + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} + + - name: Login to ghcr.io + uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3.7.0 + with: + registry: ghcr.io + username: ${{ github.repository_owner }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Create and push multi-arch manifests + env: + TAG: ${{ env.RELEASE_TAG }} + MOVING: ${{ env.MOVING_TAG }} + run: | + set -e + # $MOVING is deliberately unquoted: it is empty for a push to main, + # and an empty word would create a manifest named "$repo:". + for repo in "$DOCKERHUB_NAMESPACE/docsgpt-sandbox" "ghcr.io/${{ github.repository_owner }}/docsgpt-sandbox"; do + for name in "$TAG" $MOVING; do + docker manifest create "$repo:$name" \ + --amend "$repo:$TAG-amd64" \ + --amend "$repo:$TAG-arm64" + docker manifest push "$repo:$name" + done + done diff --git a/deployment/k8s/deployments/sandbox-deploy.yaml b/deployment/k8s/deployments/sandbox-deploy.yaml index ca82601b..d917cc1c 100644 --- a/deployment/k8s/deployments/sandbox-deploy.yaml +++ b/deployment/k8s/deployments/sandbox-deploy.yaml @@ -21,6 +21,12 @@ # sibling kernels or bypass the session cap). The gateway fails closed if the # token is unset. # +# The image is built from deployment/sandbox and published as +# arc53/docsgpt-sandbox (also ghcr.io/arc53/docsgpt-sandbox) by the release +# workflow, with `develop` tracking main. It is pulled here by the floating +# `latest` tag: pin it to a release tag if you would rather not pick up a new +# runner runtime on a pod restart. +# # On Linux prod, schedule this onto a gVisor `runsc` RuntimeClass for kernel # isolation (uncomment `runtimeClassName` once the node has it installed). # diff --git a/docs/content/_meta.js b/docs/content/_meta.js index c5e71a9f..759417a6 100644 --- a/docs/content/_meta.js +++ b/docs/content/_meta.js @@ -13,8 +13,5 @@ export default { "href": "https://gptcloud.arc53.com/" }, "Guides": "Guides", - "changelog": { - "title": "Changelog", - "display": "hidden" - } + "changelog": "Changelog" } diff --git a/docs/content/changelog.mdx b/docs/content/changelog.mdx index 504854f3..9defbbd1 100644 --- a/docs/content/changelog.mdx +++ b/docs/content/changelog.mdx @@ -1,3 +1,93 @@ --- title: 'Changelog' ---- \ No newline at end of file +description: What changed in each DocsGPT release, with links to the upgrade steps an existing deployment needs. +--- + +import { Callout } from 'nextra/components' + +# Changelog + +The notable changes in each release. Every release on GitHub also carries +[auto-generated notes](https://github.com/arc53/DocsGPT/releases) listing every merged pull +request, and [Upgrading](/upgrading) covers the steps an existing deployment has to take. + +## 0.20.0 + +### DocsGPT installs from PyPI + +The backend is published as [`docsgpt`](https://pypi.org/project/docsgpt/): the API server, the +web UI, the Celery worker and the maintenance scripts in one package, behind a single `docsgpt` +command. `pip install docsgpt` then `docsgpt api` serves the API and the UI on one port, and +`docsgpt worker` runs the worker. The optional engines are extras, `docsgpt[docling]` and +`docsgpt[milvus]`. See [Install with pip](/Deploying/Pip-Install). + +### The Python package is now `docsgpt` + +The import package was renamed from `application` to `docsgpt`, the name it has on PyPI. Entry +points move with it: `celery -A docsgpt.app.celery worker` and +`uvicorn docsgpt.asgi:asgi_app`. The old spellings still run for this release and print a +`FutureWarning`, and Celery tasks queued under the old names are still consumed. The +[upgrade guide](/upgrading) has the details. + +### Smaller default install and images + +The document and vector-store engines that pulled the heaviest dependencies are now extras +rather than defaults, so a stock install no longer carries PyTorch or the CUDA stack. The +published image follows the same split: the default `arc53/docsgpt` image is the slim one, and +`arc53/docsgpt:-docling` bakes in the docling parser engine, its models and tesseract +for OCR. + +### Faster embeddings, pinned to your installation + +Embeddings run through [FastEmbed](https://github.com/qdrant/fastembed) on ONNX Runtime. Query +embedding now happens on the Celery worker rather than in the API process, which keeps the API +small; a worker consuming the `embeddings` queue is required for search. The embedding model is +pinned to the installation instead of following the release, so an upgrade never silently +changes the model behind an existing index, and a query against an index built with a different +model is now warned about. New installs default to +`ibm-granite/granite-embedding-311m-multilingual-r2`, which is multilingual with a 32k-token +context. The `reembed` script rebuilds vectors in place from the chunk text already stored. + + + If you start your worker with an explicit `-Q`, add the `embeddings` queue. See + [Upgrading](/upgrading) for the one-line change and what happens without it. + + +### Broader document support + +Parsing gained support for more document, spreadsheet, presentation, EPUB, XHTML and image +formats, along with a reworked OCR path. Parsing behaviour is configurable: markdown conversion, +structured output, table reconstruction and improved PDF handling. + +### Agents and workflows + +Agents no longer require a source. The synthetic "Default" source is gone, and an agent with no +source or retriever is a valid agent that lists and answers normally. Workflow agents can be +exported and imported as files, so a workflow can move between deployments or into version +control. + +Artifact and tool-call handling was hardened throughout: durable tasks retry rather than lose +work on resume, tool calls are validated more strictly, and sandbox sessions are steadier. + +### Chat and attachments + +Cross-turn chaining against the Responses API is now bounded, conversation compression persists +across turns instead of being recomputed, and requests carry prompt-cache hints. Attachments +carry provenance through parsing, unparseable chat attachments are refused at upload rather than +failing mid-answer, and the composer guards against sending while an attachment is still +processing. + +### Widget + +The React widget got a UI refresh and an expand and collapse toggle, and was brought back in +line with the current API contract. The npm packages `docsgpt` and `docsgpt-react` are published +at 0.7.1. + +### Also in this release + +- The code-execution sandbox runner is published as `arc53/docsgpt-sandbox`, so the Kubernetes + manifest no longer needs an image you build yourself. +- Releases publish the Docker images and the PyPI package from the release workflow. +- The [architecture guide](/Guides/Architecture) was rewritten. +- Markdown code spans are no longer corrupted by citation rendering. +- `setup.ps1` runs on Windows PowerShell 5.1 again.