Merge branch 'arc53:main' into main

This commit is contained in:
Manish Madan authored and GitHub committed 2026-09-08 04:52:03 +05:30
commit 5c00da93be
1006 files changed
+21714 -10075

No files matched your search

+3 -3
View File
@@ -19,17 +19,17 @@ Run the full app under uvicorn (serves `/mcp` and the async SSE reconnect
routes, and matches production):
```bash
uvicorn application.asgi:asgi_app --host 0.0.0.0 --port 7091 --reload
uvicorn docsgpt.asgi:asgi_app --host 0.0.0.0 --port 7091 --reload
```
`flask --app application/app.py run --host=0.0.0.0 --port=7091` is faster but
`flask --app docsgpt/app.py run --host=0.0.0.0 --port=7091` is faster but
serves only the WSGI Flask app — it omits `/mcp` and the reconnect reader
`GET /api/messages/<id>/events`, so a dropped stream won't auto-resume.
### Celery (Task Queue)
```bash
celery -A application.app.celery worker -l INFO -Q docsgpt,parsing,embeddings
celery -A docsgpt.app.celery worker -l INFO -Q docsgpt,parsing,embeddings
```
The `parsing` queue serves document parsing (the `read_document` tool / workflow
+2 -2
View File
@@ -22,8 +22,8 @@ fi
# The embedding model is fetched on first use and cached, so nothing to download
# here. For an offline container, run `python -m application.scripts.prefetch_models`
# here. For an offline container, run `python -m docsgpt.scripts.prefetch_models`
# after the install below.
pip install -r application/requirements.txt
pip install -r docsgpt/requirements.txt
cd frontend
npm install --include=dev
+25
View File
@@ -0,0 +1,25 @@
# Build context for docsgpt/Dockerfile is the repository root, so the image can
# carry the `application` import alias next to the `docsgpt` package. Allow only
# what the image needs; everything else (frontend, docs, tests, venvs) stays out.
*
!docsgpt/
# Only the alias file: an upgraded checkout may still hold gitignored
# application/{inputs,indexes,vectors,.env} from the old layout.
!application/__init__.py
# Inside the package: caches, local runtime data and secrets never ship.
**/__pycache__/
**/*.py[cod]
docsgpt/.pytest_cache/
docsgpt/.ruff_cache/
docsgpt/.coverage
docsgpt/htmlcov/
docsgpt/*.log
docsgpt/indexes/
docsgpt/inputs/
docsgpt/vectors/
docsgpt/*.faiss
docsgpt/*.pkl
docsgpt/.env
docsgpt/.env.*
docsgpt/Dockerfile
+1 -1
View File
@@ -21,7 +21,7 @@ INTERNAL_KEY=<internal key for worker-to-backend authentication>
# searching a different vector space than the stored vectors -- which fails
# silently, because both models are 768-dimensional. To switch, set it and then
# run:
# python -m application.scripts.reembed
# python -m docsgpt.scripts.reembed
# EMBEDDINGS_NAME=ibm-granite/granite-embedding-311m-multilingual-r2
# Remote Embeddings (Optional - for using a remote embeddings API instead of
+1 -1
View File
@@ -10,7 +10,7 @@ DocsGPT ingests content (files/URLs/connectors), indexes it, and answers queries
Core components:
- Backend API (`application/`)
- Workers/ingestion (`application/worker.py` and related modules)
- Workers/ingestion (`docsgpt/worker.py` and related modules)
- Datastores (MongoDB/Redis/vector stores)
- Frontend (`frontend/`)
- Optional extensions/integrations (`extensions/`)
+1 -1
View File
@@ -6,7 +6,7 @@
version: 2
updates:
- package-ecosystem: "pip" # See documentation for possible values
directory: "/application" # Location of package manifests
directory: "/docsgpt" # Location of package manifests
schedule:
interval: "daily"
- package-ecosystem: "npm" # See documentation for possible values
+1 -1
View File
@@ -8,7 +8,7 @@ github:
application:
- changed-files:
- any-glob-to-any-file: 'application/**/*'
- any-glob-to-any-file: 'docsgpt/**/*'
docs:
- changed-files:
+4 -4
View File
@@ -4,7 +4,7 @@ on:
push:
branches: [main]
paths:
- 'application/version.py'
- 'docsgpt/version.py'
workflow_dispatch:
permissions:
@@ -23,12 +23,12 @@ jobs:
with:
persist-credentials: false
- name: Read version from application/version.py
- name: Read version from docsgpt/version.py
id: ver
run: |
VERSION=$(python3 -c "g={}; exec(open('application/version.py').read(), g); print(g['__version__'])")
VERSION=$(python3 -c "g={}; exec(open('docsgpt/version.py').read(), g); print(g['__version__'])")
if [ -z "$VERSION" ]; then
echo "::error::Could not read __version__ from application/version.py"
echo "::error::Could not read __version__ from docsgpt/version.py"
exit 1
fi
echo "version=$VERSION" >> "$GITHUB_OUTPUT"
+2 -2
View File
@@ -28,13 +28,13 @@ jobs:
run: |
python -m pip install --upgrade pip
pip install bandit # Bandit is needed for this action
if [ -f application/requirements.txt ]; then pip install -r application/requirements.txt; fi
if [ -f docsgpt/requirements.txt ]; then pip install -r docsgpt/requirements.txt; fi
- name: Run Bandit scan
uses: PyCQA/bandit-action@v1
with:
severity: medium
confidence: medium
targets: application/
targets: docsgpt/
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+83 -45
View File
@@ -7,106 +7,144 @@ on:
jobs:
build:
if: github.repository == 'arc53/DocsGPT'
# Publishing jobs run in a GitHub Actions environment so the registry
# credentials can be scoped to it and protection rules (required reviewers,
# branch restrictions) applied in the repository settings.
environment: docker-hub
env:
# Public namespace the compose files pull from; the login secret only
# authenticates the push.
DOCKERHUB_NAMESPACE: arc53
strategy:
matrix:
include:
- platform: linux/amd64
runner: ubuntu-latest
suffix: amd64
- platform: linux/arm64
runner: ubuntu-24.04-arm
suffix: arm64
runs-on: ${{ matrix.runner }}
platform: [linux/amd64, linux/arm64]
# "" is the slim default image; "-docling" bakes the docling parser
# engine, its models and tesseract in (OCR-ready).
variant: ["", "-docling"]
runs-on: ${{ matrix.platform == 'linux/arm64' && 'ubuntu-24.04-arm' || 'ubuntu-latest' }}
permissions:
contents: read
packages: write
steps:
- uses: actions/checkout@v4
- name: Set up QEMU # Only needed for emulation, not for native arm64 builds
if: matrix.platform == 'linux/arm64'
uses: docker/setup-qemu-action@v3
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
with:
persist-credentials: false
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3.12.0
with:
driver: docker-container
install: true
- name: Login to DockerHub
uses: docker/login-action@v3
uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3.7.0
with:
username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.DOCKER_PASSWORD }}
- name: Login to ghcr.io
uses: docker/login-action@v3
uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3.7.0
with:
registry: ghcr.io
username: ${{ github.repository_owner }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Build and push platform-specific images
uses: docker/build-push-action@v6
- name: Image metadata (OCI labels)
id: meta
uses: docker/metadata-action@c299e40c65443455700f0fdfc63efafe5b349051 # v5.10.0
with:
file: './application/Dockerfile'
images: |
${{ env.DOCKERHUB_NAMESPACE }}/docsgpt
ghcr.io/${{ github.repository_owner }}/docsgpt
labels: |
org.opencontainers.image.title=DocsGPT${{ matrix.variant }}
org.opencontainers.image.version=${{ github.event.release.tag_name }}
- name: Build and push platform-specific images
uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6.19.2
with:
file: './docsgpt/Dockerfile'
platforms: ${{ matrix.platform }}
context: ./application
context: .
push: true
build-args: |
EXTRAS=${{ matrix.variant == '-docling' && 'docling' || '' }}
INSTALL_TESSERACT=${{ matrix.variant == '-docling' && 'true' || 'false' }}
tags: |
${{ secrets.DOCKER_USERNAME }}/docsgpt:${{ github.event.release.tag_name }}-${{ matrix.suffix }}
ghcr.io/${{ github.repository_owner }}/docsgpt:${{ github.event.release.tag_name }}-${{ matrix.suffix }}
${{ env.DOCKERHUB_NAMESPACE }}/docsgpt:${{ github.event.release.tag_name }}${{ matrix.variant }}-${{ matrix.platform == 'linux/arm64' && 'arm64' || 'amd64' }}
ghcr.io/${{ github.repository_owner }}/docsgpt:${{ github.event.release.tag_name }}${{ matrix.variant }}-${{ matrix.platform == 'linux/arm64' && 'arm64' || 'amd64' }}
labels: ${{ steps.meta.outputs.labels }}
provenance: false
sbom: false
cache-from: type=registry,ref=${{ secrets.DOCKER_USERNAME }}/docsgpt:latest
cache-from: type=registry,ref=${{ env.DOCKERHUB_NAMESPACE }}/docsgpt:latest${{ matrix.variant }}
cache-to: type=inline
manifest:
if: github.repository == 'arc53/DocsGPT'
# Publishing jobs run in a GitHub Actions environment so the registry
# credentials can be scoped to it and protection rules (required reviewers,
# branch restrictions) applied in the repository settings.
environment: docker-hub
env:
# Public namespace the compose files pull from; the login secret only
# authenticates the push.
DOCKERHUB_NAMESPACE: arc53
needs: build
strategy:
matrix:
variant: ["", "-docling"]
runs-on: ubuntu-latest
permissions:
packages: write
steps:
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3.12.0
with:
driver: docker-container
install: true
- name: Login to DockerHub
uses: docker/login-action@v3
uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3.7.0
with:
username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.DOCKER_PASSWORD }}
- name: Login to ghcr.io
uses: docker/login-action@v3
uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3.7.0
with:
registry: ghcr.io
username: ${{ github.repository_owner }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Create and push manifest for DockerHub
- name: Create and push multi-arch manifests
env:
TAG: ${{ github.event.release.tag_name }}${{ matrix.variant }}
LATEST: latest${{ matrix.variant }}
run: |
set -e
docker manifest create ${{ secrets.DOCKER_USERNAME }}/docsgpt:${{ github.event.release.tag_name }} \
--amend ${{ secrets.DOCKER_USERNAME }}/docsgpt:${{ github.event.release.tag_name }}-amd64 \
--amend ${{ secrets.DOCKER_USERNAME }}/docsgpt:${{ github.event.release.tag_name }}-arm64
docker manifest push ${{ secrets.DOCKER_USERNAME }}/docsgpt:${{ github.event.release.tag_name }}
docker manifest create ${{ secrets.DOCKER_USERNAME }}/docsgpt:latest \
--amend ${{ secrets.DOCKER_USERNAME }}/docsgpt:${{ github.event.release.tag_name }}-amd64 \
--amend ${{ secrets.DOCKER_USERNAME }}/docsgpt:${{ github.event.release.tag_name }}-arm64
docker manifest push ${{ secrets.DOCKER_USERNAME }}/docsgpt:latest
for repo in "$DOCKERHUB_NAMESPACE/docsgpt" "ghcr.io/${{ github.repository_owner }}/docsgpt"; do
for name in "$TAG" "$LATEST"; do
docker manifest create "$repo:$name" \
--amend "$repo:$TAG-amd64" \
--amend "$repo:$TAG-arm64"
docker manifest push "$repo:$name"
done
done
- name: Create and push manifest for ghcr.io
release-assets:
if: github.repository == 'arc53/DocsGPT'
needs: manifest
runs-on: ubuntu-latest
permissions:
contents: write
steps:
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
with:
persist-credentials: false
- name: Attach the standalone compose file to the release
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
TAG: ${{ github.event.release.tag_name }}
run: |
set -e
docker manifest create ghcr.io/${{ github.repository_owner }}/docsgpt:${{ github.event.release.tag_name }} \
--amend ghcr.io/${{ github.repository_owner }}/docsgpt:${{ github.event.release.tag_name }}-amd64 \
--amend ghcr.io/${{ github.repository_owner }}/docsgpt:${{ github.event.release.tag_name }}-arm64
docker manifest push ghcr.io/${{ github.repository_owner }}/docsgpt:${{ github.event.release.tag_name }}
docker manifest create ghcr.io/${{ github.repository_owner }}/docsgpt:latest \
--amend ghcr.io/${{ github.repository_owner }}/docsgpt:${{ github.event.release.tag_name }}-amd64 \
--amend ghcr.io/${{ github.repository_owner }}/docsgpt:${{ github.event.release.tag_name }}-arm64
docker manifest push ghcr.io/${{ github.repository_owner }}/docsgpt:latest
gh release upload "$TAG" deployment/docker-compose-standalone.yaml --clobber
+67 -36
View File
@@ -9,92 +9,123 @@ on:
jobs:
build:
if: github.repository == 'arc53/DocsGPT'
# Publishing jobs run in a GitHub Actions environment so the registry
# credentials can be scoped to it and protection rules (required reviewers,
# branch restrictions) applied in the repository settings.
environment: docker-hub
env:
# Public namespace the compose files pull from; the login secret only
# authenticates the push.
DOCKERHUB_NAMESPACE: arc53
strategy:
matrix:
include:
- platform: linux/amd64
runner: ubuntu-latest
suffix: amd64
- platform: linux/arm64
runner: ubuntu-24.04-arm
suffix: arm64
runs-on: ${{ matrix.runner }}
platform: [linux/amd64, linux/arm64]
# "" is the slim default image; "-docling" bakes the docling parser
# engine, its models and tesseract in (OCR-ready).
variant: ["", "-docling"]
runs-on: ${{ matrix.platform == 'linux/arm64' && 'ubuntu-24.04-arm' || 'ubuntu-latest' }}
permissions:
contents: read
packages: write
steps:
- uses: actions/checkout@v4
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
with:
persist-credentials: false
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3.12.0
with:
driver: docker-container
install: true
- name: Login to DockerHub
uses: docker/login-action@v3
uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3.7.0
with:
username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.DOCKER_PASSWORD }}
- name: Login to ghcr.io
uses: docker/login-action@v3
uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3.7.0
with:
registry: ghcr.io
username: ${{ github.repository_owner }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Build and push platform-specific images
uses: docker/build-push-action@v6
- name: Image metadata (OCI labels)
id: meta
uses: docker/metadata-action@c299e40c65443455700f0fdfc63efafe5b349051 # v5.10.0
with:
file: './application/Dockerfile'
images: |
${{ env.DOCKERHUB_NAMESPACE }}/docsgpt
ghcr.io/${{ github.repository_owner }}/docsgpt
labels: |
org.opencontainers.image.title=DocsGPT${{ matrix.variant }}
org.opencontainers.image.version=develop
- name: Build and push platform-specific images
uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6.19.2
with:
file: './docsgpt/Dockerfile'
platforms: ${{ matrix.platform }}
context: ./application
context: .
push: true
build-args: |
EXTRAS=${{ matrix.variant == '-docling' && 'docling' || '' }}
INSTALL_TESSERACT=${{ matrix.variant == '-docling' && 'true' || 'false' }}
tags: |
${{ secrets.DOCKER_USERNAME }}/docsgpt:develop-${{ matrix.suffix }}
ghcr.io/${{ github.repository_owner }}/docsgpt:develop-${{ matrix.suffix }}
${{ env.DOCKERHUB_NAMESPACE }}/docsgpt:develop${{ matrix.variant }}-${{ matrix.platform == 'linux/arm64' && 'arm64' || 'amd64' }}
ghcr.io/${{ github.repository_owner }}/docsgpt:develop${{ matrix.variant }}-${{ matrix.platform == 'linux/arm64' && 'arm64' || 'amd64' }}
labels: ${{ steps.meta.outputs.labels }}
provenance: false
sbom: false
cache-from: type=registry,ref=${{ secrets.DOCKER_USERNAME }}/docsgpt:develop
cache-from: type=registry,ref=${{ env.DOCKERHUB_NAMESPACE }}/docsgpt:develop${{ matrix.variant }}
cache-to: type=inline
manifest:
if: github.repository == 'arc53/DocsGPT'
# Publishing jobs run in a GitHub Actions environment so the registry
# credentials can be scoped to it and protection rules (required reviewers,
# branch restrictions) applied in the repository settings.
environment: docker-hub
env:
# Public namespace the compose files pull from; the login secret only
# authenticates the push.
DOCKERHUB_NAMESPACE: arc53
needs: build
strategy:
matrix:
variant: ["", "-docling"]
runs-on: ubuntu-latest
permissions:
packages: write
steps:
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3.12.0
with:
driver: docker-container
install: true
- name: Login to DockerHub
uses: docker/login-action@v3
uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3.7.0
with:
username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.DOCKER_PASSWORD }}
- name: Login to ghcr.io
uses: docker/login-action@v3
uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3.7.0
with:
registry: ghcr.io
username: ${{ github.repository_owner }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Create and push manifest for DockerHub
run: |
docker manifest create ${{ secrets.DOCKER_USERNAME }}/docsgpt:develop \
--amend ${{ secrets.DOCKER_USERNAME }}/docsgpt:develop-amd64 \
--amend ${{ secrets.DOCKER_USERNAME }}/docsgpt:develop-arm64
docker manifest push ${{ secrets.DOCKER_USERNAME }}/docsgpt:develop
- name: Create and push manifest for ghcr.io
- name: Create and push multi-arch manifests
env:
TAG: develop${{ matrix.variant }}
run: |
docker manifest create ghcr.io/${{ github.repository_owner }}/docsgpt:develop \
--amend ghcr.io/${{ github.repository_owner }}/docsgpt:develop-amd64 \
--amend ghcr.io/${{ github.repository_owner }}/docsgpt:develop-arm64
docker manifest push ghcr.io/${{ github.repository_owner }}/docsgpt:develop
set -e
for repo in "$DOCKERHUB_NAMESPACE/docsgpt" "ghcr.io/${{ github.repository_owner }}/docsgpt"; do
docker manifest create "$repo:$TAG" \
--amend "$repo:$TAG-amd64" \
--amend "$repo:$TAG-arm64"
docker manifest push "$repo:$TAG"
done
+67
View File
@@ -0,0 +1,67 @@
name: Verify the Docker image works offline
# Builds the backend image and runs its offline check with networking off, so
# a change that reintroduces a first-request download (a tokenizer, tiktoken's
# encoding, an embedding model) fails here instead of in an air-gapped install.
on:
workflow_dispatch:
pull_request:
paths:
- 'docsgpt/Dockerfile'
- '.dockerignore'
- 'application/**'
- 'docsgpt/requirements*.txt'
- 'docsgpt/scripts/prefetch_models.py'
- 'docsgpt/scripts/verify_offline.py'
- 'docsgpt/vectorstore/model_registry.py'
- 'docsgpt/parser/tokenization.py'
- 'docsgpt/vectorstore/embeddings_local.py'
- '.github/workflows/docker-image-verify.yml'
permissions:
contents: read
jobs:
verify:
strategy:
matrix:
# "" is the slim default; "-docling" bakes docling, its models and
# tesseract in, so the conversion check in verify_offline runs too.
variant: ["", "-docling"]
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
with:
persist-credentials: false
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3.12.0
- name: Build the image
uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6.19.2
with:
file: ./docsgpt/Dockerfile
context: .
platforms: linux/amd64
load: true
tags: docsgpt:verify${{ matrix.variant }}
build-args: |
EXTRAS=${{ matrix.variant == '-docling' && 'docling' || '' }}
INSTALL_TESSERACT=${{ matrix.variant == '-docling' && 'true' || 'false' }}
cache-from: type=gha,scope=verify${{ matrix.variant }}
cache-to: type=gha,mode=max,scope=verify${{ matrix.variant }}
- name: Image size
env:
IMAGE: docsgpt:verify${{ matrix.variant }}
run: |
docker image inspect "$IMAGE" --format '{{.Size}}' | awk '{printf "uncompressed: %.2f GB\n", $1/1e9}'
docker history "$IMAGE" --format '{{.Size}}\t{{.CreatedBy}}' | head -20
- name: Offline verification (no network)
env:
IMAGE: docsgpt:verify${{ matrix.variant }}
run: |
docker run --rm --network none "$IMAGE" \
python -m docsgpt.scripts.verify_offline
+17
View File
@@ -20,3 +20,20 @@ jobs:
uses: chartboost/ruff-action@v1
with:
version: 0.14.10
requirements-in-sync:
# application/requirements*.txt are exported from uv.lock; fail when a
# change to pyproject.toml or uv.lock was not re-exported.
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
with:
persist-credentials: false
- uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6.8.0
- name: Re-export and diff
run: |
uv lock --check
bash scripts/export_requirements.sh
git diff --exit-code -- docsgpt/requirements.txt docsgpt/requirements-docling.txt docsgpt/requirements-milvus.txt
+2 -2
View File
@@ -20,13 +20,13 @@ jobs:
- name: Install dependencies
run: |
python -m pip install --upgrade pip
cd application
cd docsgpt
if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
cd ../tests
if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
- name: Test with pytest and generate coverage report
run: |
python -m pytest -n auto --cov=application --cov-report=xml --cov-report=term-missing
python -m pytest -n auto --cov=docsgpt --cov-report=xml --cov-report=term-missing
- name: Upload coverage reports to Codecov
if: github.event_name == 'pull_request' && matrix.python-version == '3.12'
uses: codecov/codecov-action@v5
+2 -1
View File
@@ -179,6 +179,7 @@ frontend/*.njsproj
frontend/*.sln
frontend/*.sw?
docsgpt/vectors/
application/vectors/
**/inputs
@@ -202,6 +203,6 @@ tests/e2e/node_modules/
tests/e2e/playwright-report/
tests/e2e/test-results/
tests/e2e/.e2e-last-run.json
application/core/models/foundry_test_internal.yaml
docsgpt/core/models/foundry_test_internal.yaml
bench/
/benchmark/
+2 -2
View File
@@ -14,7 +14,7 @@
"request": "launch",
"module": "flask",
"env": {
"FLASK_APP": "application/app.py",
"FLASK_APP": "docsgpt/app.py",
"PYTHONPATH": "${workspaceFolder}",
"FLASK_ENV": "development",
"FLASK_DEBUG": "1",
@@ -38,7 +38,7 @@
},
"args": [
"-A",
"application.app.celery",
"docsgpt.app.celery",
"worker",
"-l",
"INFO",
+26 -16
View File
@@ -28,22 +28,32 @@ Use these commands once the dev prerequisites above are satisfied.
```bash
source .venv/bin/activate # macOS/Linux
uv pip install -r application/requirements.txt # or: pip install -r application/requirements.txt
# Optional docling parser engine (OCR / read_document structured output):
# uv pip install -r application/requirements-docling.txt
uv pip install -r docsgpt/requirements.txt # or: pip install -r docsgpt/requirements.txt
# Optional extras (not installed by default; each file = core + the extra):
# uv pip install -r docsgpt/requirements-docling.txt # docling parser engine (OCR backend, structured output)
# uv pip install -r docsgpt/requirements-milvus.txt # VECTOR_STORE=milvus
# With uv alone: `uv sync --extra docling` (pyproject.toml + uv.lock are the source of truth).
# `uv pip install -r docsgpt/requirements-docling.txt` needs UV_INDEX_STRATEGY=unsafe-best-match
# (the file adds the PyTorch CPU index; prefer `uv sync --extra docling`).
```
Dependencies are declared in `pyproject.toml` and locked in `uv.lock`; the
`docsgpt/requirements*.txt` files are exported from the lock. To add or
bump a package: edit `pyproject.toml`, run `uv lock`, then
`bash scripts/export_requirements.sh` (CI fails if the exports are stale).
Never edit the requirements files by hand.
Run the API. For local dev, prefer the ASGI entrypoint under uvicorn — it
serves the **whole** app, matches production, and hot-reloads:
```bash
uvicorn application.asgi:asgi_app --host 0.0.0.0 --port 7091 --reload
uvicorn docsgpt.asgi:asgi_app --host 0.0.0.0 --port 7091 --reload
```
`flask --app application/app.py run --host=0.0.0.0 --port=7091` is a faster
`flask --app docsgpt/app.py run --host=0.0.0.0 --port=7091` is a faster
inner loop (quick startup, the Werkzeug interactive debugger), but it serves
**only** the WSGI Flask app and omits the routes mounted on the ASGI shell
in `application/asgi.py`:
in `docsgpt/asgi.py`:
- the `/mcp` FastMCP endpoint, and
- the native-async SSE reconnect reader `GET /api/messages/<id>/events`.
@@ -53,13 +63,13 @@ Flask route), but a stream interrupted by a disconnect won't auto-resume on
reconnect. Use `flask run` only when you don't need those routes.
Production uses `gunicorn -k uvicorn_worker.UvicornWorker` against the same
`application.asgi:asgi_app` target; see `application/Dockerfile` for the
`docsgpt.asgi:asgi_app` target; see `docsgpt/Dockerfile` for the
full flag set.
Run the Celery worker in a separate terminal:
```bash
celery -A application.app.celery worker -l INFO
celery -A docsgpt.app.celery worker -l INFO
```
**The worker is required for retrieval, not optional.** `EMBEDDINGS_DELEGATE_TO_WORKER`
@@ -73,7 +83,7 @@ loading a model of its own — which keeps the API process around 285 MB instead
On macOS, prefer the solo pool for Celery:
```bash
python -m celery -A application.app.celery worker -l INFO --pool=solo
python -m celery -A docsgpt.app.celery worker -l INFO --pool=solo
```
Note that `--pool=solo` costs roughly 350 ms per query embed against ~55 ms on the
@@ -147,7 +157,7 @@ vale .
## Repository map
- `application/`: Flask backend, API routes, agent logic, retrieval, parsing, security, storage, Celery worker, and WSGI entrypoints.
- `docsgpt/`: Flask backend, API routes, agent logic, retrieval, parsing, security, storage, Celery worker, and WSGI entrypoints.
- `tests/`: backend unit/integration tests and test-only Python dependencies.
- `frontend/`: Vite + React + TypeScript application.
- `frontend/src/`: main UI code, including `components`, `conversation`, `hooks`, `locale`, `settings`, `upload`, and Redux store wiring in `store.ts`.
@@ -167,12 +177,12 @@ vale .
### Backend Abstractions
- LLM providers implement a common interface in `application/llm/` (add new providers by extending the base class).
- Vector stores are abstracted in `application/vectorstore/`.
- Parsers live in `application/parser/` and handle different document formats in the ingestion stage.
- Agents and tools are in `application/agents/` and `application/agents/tools/`.
- Celery setup/config lives in `application/celery_init.py` and `application/celeryconfig.py`.
- Settings and env vars are managed via Pydantic in `application/core/settings.py`.
- LLM providers implement a common interface in `docsgpt/llm/` (add new providers by extending the base class).
- Vector stores are abstracted in `docsgpt/vectorstore/`.
- Parsers live in `docsgpt/parser/` and handle different document formats in the ingestion stage.
- Agents and tools are in `docsgpt/agents/` and `docsgpt/agents/tools/`.
- Celery setup/config lives in `docsgpt/celery_init.py` and `docsgpt/celeryconfig.py`.
- Settings and env vars are managed via Pydantic in `docsgpt/core/settings.py`.
### Frontend
+1 -1
View File
@@ -49,7 +49,7 @@ Tech Stack Overview:
### 🖥 Backend Contributions (🐍 Python)
- Review our issues and contribute to [`/application`](https://github.com/arc53/DocsGPT/tree/main/application)
- Review our issues and contribute to [`/docsgpt`](https://github.com/arc53/DocsGPT/tree/main/docsgpt)
- All new code should be covered with unit tests ([pytest](https://github.com/pytest-dev/pytest)). Please find tests under [`/tests`](https://github.com/arc53/DocsGPT/tree/main/tests) folder.
- Before submitting your Pull Request, ensure it can be queried after ingesting some test data.
- **Coding Style:** We adhere to the [PEP 8](https://www.python.org/dev/peps/pep-0008/) style guide for Python code. We use `ruff` as our linter and code formatter. Please ensure your code is formatted correctly and passes `ruff` checks before submitting.
+1 -1
View File
@@ -132,7 +132,7 @@ Please refer to the [CONTRIBUTING.md](CONTRIBUTING.md) file for information abou
## Project Structure
- **Application** - Backend Flask application.
- **docsgpt** - Backend Flask application (the `docsgpt` Python package).
- **Extensions** - Integrations and widgets (e.g., Chatwoot, React widget).
-148
View File
@@ -1,148 +0,0 @@
# Builder Stage
FROM ubuntu:24.04 as builder
ENV DEBIAN_FRONTEND=noninteractive
RUN apt-get update && \
apt-get install -y software-properties-common && \
add-apt-repository ppa:deadsnakes/ppa && \
apt-get update && \
apt-get install -y --no-install-recommends gcc g++ wget unzip libc6-dev python3.12 python3.12-venv python3.12-dev && \
rm -rf /var/lib/apt/lists/*
# Verify Python installation and setup symlink
RUN if [ -f /usr/bin/python3.12 ]; then \
ln -s /usr/bin/python3.12 /usr/bin/python; \
else \
echo "Python 3.12 not found"; exit 1; \
fi
# Install Rust
RUN wget -q -O - https://sh.rustup.rs | sh -s -- -y
# Clean up to reduce container size
RUN apt-get remove --purge -y wget unzip && apt-get autoremove -y && rm -rf /var/lib/apt/lists/*
# Copy requirements manifests
COPY requirements.txt requirements-docling.txt ./
# Setup Python virtual environment
RUN python3.12 -m venv /venv
# Activate virtual environment and install Python packages
ENV PATH="/venv/bin:$PATH"
# Install Python packages
RUN pip install --no-cache-dir --upgrade pip && \
pip install --no-cache-dir tiktoken && \
pip install --no-cache-dir -r requirements.txt
# Optional docling parser engine (DOC_PARSER_ENGINE=docling, the docling OCR
# backend, read_document's structured output) — OFF by default: it pulls the
# layout/OCR model stack and adds gigabytes to the image. anydoc (in
# requirements.txt) is the default parser and needs none of it, and OCR runs
# natively on tesseract (below, also opt-in) without it.
ARG INSTALL_DOCLING=false
RUN if [ "$INSTALL_DOCLING" = "true" ]; then \
pip install --no-cache-dir -r requirements-docling.txt; \
fi
# Final Stage
FROM ubuntu:24.04 as final
RUN apt-get update && \
apt-get install -y software-properties-common && \
add-apt-repository ppa:deadsnakes/ppa && \
apt-get update && apt-get install -y --no-install-recommends \
python3.12 \
libgl1 \
libglib2.0-0 \
poppler-utils \
&& \
ln -s /usr/bin/python3.12 /usr/bin/python && \
rm -rf /var/lib/apt/lists/*
# Optional tesseract OCR engine (OCR_ENABLED=true with OCR_ENGINE=tesseract,
# the default engine) — OFF by default like every other OCR dependency; OCR
# itself is off unless configured. Opt in with --build-arg
# INSTALL_TESSERACT=true (setup.sh writes it to .env when OCR is enabled);
# ~35 MB of system packages. Extra language packs are a deployment concern
# (apt: tesseract-ocr-<lang>, then list them in OCR_LANGS). A DeepSeek-OCR
# endpoint (OCR_ENGINE=deepseek) needs none of this.
ARG INSTALL_TESSERACT=false
RUN if [ "$INSTALL_TESSERACT" = "true" ]; then \
apt-get update && \
apt-get install -y --no-install-recommends tesseract-ocr tesseract-ocr-eng && \
rm -rf /var/lib/apt/lists/*; \
fi
# Set working directory
WORKDIR /app
# Create a non-root user: `appuser` (Feel free to choose a name)
RUN groupadd -r appuser && \
useradd -r -g appuser -d /app -s /sbin/nologin -c "Docker image user" appuser
# Copy the virtual environment and model from the builder stage
COPY --from=builder /venv /venv
# Pre-fetch the embedding models into FastEmbed's cache so a fresh container
# does not download on first ingest and an air-gapped install works at all.
# Both defaults are baked: an upgraded deployment keeps using mpnet until it
# runs the re-embed script, while a new one starts on granite.
# The prefetch writes hub-layout snapshots (including tokenizer.json) here, so
# HF_HUB_CACHE has to point at the same directory: chunking loads the tokenizer
# through ``tokenizers``, which reads the hub cache and would otherwise fetch
# over the network on first ingest -- and fall back to cl100k when offline.
ENV EMBEDDINGS_CACHE_DIR=/app/models \
HF_HUB_CACHE=/app/models
# Only the modules the prefetch imports are copied first. It reaches nothing
# beyond model_registry, which is stdlib-only, so keeping the full source copy
# below this layer stops an unrelated edit from re-downloading ~780 MB of model
# artifacts on every build.
COPY __init__.py /app/application/__init__.py
COPY scripts/__init__.py scripts/prefetch_models.py /app/application/scripts/
COPY vectorstore/__init__.py vectorstore/model_registry.py /app/application/vectorstore/
ARG EMBEDDINGS_PREFETCH=""
RUN PYTHONPATH=/app /venv/bin/python -m application.scripts.prefetch_models ${EMBEDDINGS_PREFETCH}
# Copy your application code
COPY . /app/application
# Change the ownership of the /app directory to the appuser
RUN mkdir -p /app/application/inputs/local
RUN chown -R appuser:appuser /app
# Set environment variables
ENV FLASK_APP=app.py \
FLASK_DEBUG=true \
PATH="/venv/bin:$PATH"
ENV MALLOC_ARENA_MAX=2 \
OMP_NUM_THREADS=4 \
MKL_NUM_THREADS=4 \
OPENBLAS_NUM_THREADS=4
# Expose the port the app runs on
EXPOSE 7091
# Switch to non-root user
USER appuser
# BoundedDrainUvicornWorker makes max_requests recycles safe with held-open SSE
# connections (see application/gunicorn_worker.py); with recycles now safe,
# --max-requests is raised (kept for memory hygiene) to cut churn.
CMD ["gunicorn", \
"-w", "1", \
"-k", "application.gunicorn_worker.BoundedDrainUvicornWorker", \
"--bind", "0.0.0.0:7091", \
"--timeout", "180", \
"--graceful-timeout", "120", \
"--keep-alive", "5", \
"--worker-tmp-dir", "/dev/shm", \
"--max-requests", "5000", \
"--max-requests-jitter", "500", \
"--config", "application/gunicorn_conf.py", \
"application.asgi:asgi_app"]
+77
View File
@@ -0,0 +1,77 @@
"""``application`` is now ``docsgpt``; this alias keeps the old name importable for one release.
Every ``import application.x.y`` resolves to the already-imported ``docsgpt.x.y``
module object, so there is exactly one settings object, one Celery app and one
Flask app however a process refers to them. Entry points such as
``celery -A application.app.celery``, ``uvicorn application.asgi:asgi_app`` and
``python -m application.scripts.<name>`` keep working; update them to
``docsgpt.…`` before the alias is removed.
"""
from __future__ import annotations
import importlib
import importlib.abc
import importlib.util
import sys
import warnings
_OLD = __name__
_NEW = "docsgpt"
class _AliasLoader(importlib.abc.Loader):
"""Hand back the ``docsgpt`` module object; delegate code access to its real loader."""
def __init__(self, target: str, target_spec) -> None:
self._target = target
self._target_spec = target_spec
self._original_spec = None
def create_module(self, spec):
module = importlib.import_module(self._target)
self._original_spec = module.__spec__
return module
def exec_module(self, module) -> None:
# The import machinery stamps the alias spec on the shared module
# object; put the real one back so importlib.reload and __spec__-based
# lookups keep addressing the module by its docsgpt name.
if self._original_spec is not None:
module.__spec__ = self._original_spec
# runpy (``python -m application.x``) reads the code through the loader.
def get_code(self, fullname):
return self._target_spec.loader.get_code(self._target)
def get_source(self, fullname):
return self._target_spec.loader.get_source(self._target)
def get_filename(self, fullname):
return self._target_spec.loader.get_filename(self._target)
def is_package(self, fullname):
return self._target_spec.submodule_search_locations is not None
class _AliasFinder(importlib.abc.MetaPathFinder):
"""Resolve ``application.<path>`` to ``docsgpt.<path>``."""
def find_spec(self, name, path=None, target=None):
if name != _OLD and not name.startswith(_OLD + "."):
return None
new_name = _NEW + name[len(_OLD):]
target_spec = importlib.util.find_spec(new_name)
if target_spec is None:
return None
return importlib.util.spec_from_loader(name, _AliasLoader(new_name, target_spec))
warnings.warn(
"The 'application' package was renamed to 'docsgpt'. Update imports and entry points "
"(celery -A docsgpt.app.celery, uvicorn docsgpt.asgi:asgi_app); this alias will be removed.",
FutureWarning,
stacklevel=2,
)
sys.meta_path.insert(0, _AliasFinder())
sys.modules[_OLD] = importlib.import_module(_NEW)
-3
View File
@@ -1,3 +0,0 @@
from application.api.v1.routes import v1_bp
__all__ = ["v1_bp"]
-15
View File
@@ -1,15 +0,0 @@
# Optional docling parser engine.
#
# Needed only for: DOC_PARSER_ENGINE=docling, the docling OCR backend
# (OCR_BACKEND=docling: layout-model hybrid OCR, ocrmac/rapidocr engines),
# .adoc/.vtt/.xml attachment parsing, and the read_document tool's `structured`
# output. The default anydoc engine needs none of this, and OCR itself does
# not either: OCR_ENABLED=true with the tesseract binary (or a DeepSeek-OCR
# endpoint) runs through application/parser/file/ocr_parser.py.
#
# Install ON TOP of requirements.txt, which pins torch, transformers and
# onnxruntime (rapidocr runs on the same onnxruntime fastembed uses):
# pip install -r requirements.txt -r requirements-docling.txt
# Docker: build with --build-arg INSTALL_DOCLING=true
docling==2.119.0
rapidocr==3.9.2
-125
View File
@@ -1,125 +0,0 @@
a2wsgi==1.10.10
alembic>=1.13,<2
anthropic==0.121.0
asgiref>=3.11.1
boto3==1.43.67
beautifulsoup4==4.15.0
cel-python==0.5.0
celery==5.6.3
celery-redbeat==2.4.2
croniter==6.2.4
cryptography==50.0.0
dataclasses-json==0.6.7
daytona==0.205.1
defusedxml==0.7.1
# Default document converter (DOC_PARSER_ENGINE=anydoc). The docling engine
# (OCR, layout/table models, read_document's structured output) is an
# optional extra: pip install -r application/requirements-docling.txt
firecrawl-anydoc==0.2.3
docx2txt==0.9
ddgs>=8.0.0
fast-ebook
elevenlabs==2.62.0
Flask==3.1.3
fastembed==0.8.0
# fastembed's runtime: local embeddings execute on it, so it stays pinned in
# core even though the docling extra's rapidocr runs on the same package.
onnxruntime==1.28.0
faiss-cpu==1.15.0
fastmcp==3.4.6
flask-restx==1.3.2
google-genai==2.17.0
google-api-python-client==2.198.0
google-auth-httplib2==0.4.1
google-auth-oauthlib==1.4.0
gTTS==2.5.4
gunicorn==26.0.0
jinja2==3.1.6
jiter==0.16.0
jmespath==1.1.0
joblib==1.5.3
jsonpatch==1.33
jsonpointer==3.1.1
kombu==5.6.2
lazy-object-proxy==1.12.0
lxml==6.1.1
markupsafe==3.0.3
marshmallow>=3.18.0,<4.0.0
mpmath==1.3.0
multidict==6.7.1
msal==1.37.0
mypy-extensions==1.1.0
networkx==3.6.1
numpy==2.5.1
openai==2.53.0
openapi3-parser==1.1.22
opentelemetry-distro>=0.50b0,<1
opentelemetry-exporter-otlp>=1.29.0,<2
opentelemetry-instrumentation-celery>=0.50b0,<1
opentelemetry-instrumentation-flask>=0.50b0,<1
opentelemetry-instrumentation-logging>=0.50b0,<1
opentelemetry-instrumentation-psycopg>=0.50b0,<1
opentelemetry-instrumentation-redis>=0.50b0,<1
opentelemetry-instrumentation-requests>=0.50b0,<1
opentelemetry-instrumentation-sqlalchemy>=0.50b0,<1
opentelemetry-instrumentation-starlette>=0.50b0,<1
orjson==3.11.9
packaging==26.3
pandas==3.0.5
openpyxl==3.1.5
pathable==0.6.0
pdf2image>=1.17.0
pgvector>=0.5,<1
pillow==12.3.0
portalocker>=2.7.0,<4.0.0
prompt-toolkit==3.0.53
protobuf==7.35.1
psycopg[binary,pool]>=3.1,<4
py==1.11.0
pydantic
pydantic-core
pydantic-settings
praw==8.0.2
pymilvus==3.0.1
milvus-lite==3.2.0; sys_platform != "win32"
pypdf==6.15.0
pypdfium2==5.12.1
qdrant-client==1.19.0
python-dateutil==2.9.0.post0
python-dotenv
python-jose==3.5.0
python-pptx==1.0.2
PyYAML
redis==7.4.0
referencing>=0.28.0,<0.38.0
regex==2026.7.19
requests==2.34.2
retry==0.9.2
sqlalchemy>=2.0,<3
starlette>=1.0,<2
tiktoken==0.13.0
tokenizers==0.22.2
# torch and transformers serve docling's model stack alone -- embeddings moved
# to fastembed/onnxruntime. Still in core rather than requirements-docling.txt,
# so installing the optional extra never re-resolves the shared pins; both can
# move out with docling.
torch==2.11.0
tqdm==4.67.3
# Capped <5.9.0 to match docling-core's own darwin pin: 5.9+ breaks docling's
# PDF layout model on Apple Silicon ("Cannot convert a MPS Tensor to float64").
transformers==5.8.1
typing-extensions==4.16.0
typing-inspect==0.9.0
tzdata==2026.3
urllib3==2.7.0
uvicorn[standard]>=0.30,<1
uvicorn-worker>=0.4,<1
vine==5.1.0
wcwidth==0.8.2
werkzeug>=3.1.0
yarl==1.24.5
markdownify==1.2.3
tldextract==5.3.2
# google-genai caps websockets <17.0; 17.x will not resolve.
websockets==16.0
websocket-client==1.9.0
+32 -8
View File
@@ -1,9 +1,24 @@
services:
frontend:
build: ../frontend
build:
context: ../frontend
target: dev
environment:
# Every VITE_* the app reads. A bare name is passed through only when it is
# set in the shell or the --env-file, so an unset one does not reach the
# container as an empty string and override the image's own default.
- VITE_API_HOST=http://localhost:7091
- VITE_API_STREAMING=$VITE_API_STREAMING
- VITE_API_STREAMING=${VITE_API_STREAMING:-true}
- VITE_BASE_URL
- VITE_GOOGLE_CLIENT_ID
- VITE_GOOGLE_PICKER_API_KEY
- VITE_SHARE_POINT_CLIENT_ID
- VITE_CONFLUENCE_CLIENT_ID
- VITE_NOTIFICATION_TEXT
- VITE_NOTIFICATION_LINK
- VITE_ENABLE_VOICE_INPUT
- VITE_DISABLE_SOURCE_FE
- VITE_USE_V
ports:
- "5173:5173"
depends_on:
@@ -11,8 +26,10 @@ services:
backend:
build:
context: ../application
context: ..
dockerfile: docsgpt/Dockerfile
args:
EXTRAS: ${EXTRAS:-}
INSTALL_DOCLING: ${INSTALL_DOCLING:-false}
# Off by default; deployments running OCR_ENABLED=true with tesseract
# must set INSTALL_TESSERACT=true before rebuilding (see docker-compose.yaml).
@@ -27,9 +44,14 @@ services:
ports:
- "7091:7091"
volumes:
- ../application/indexes:/app/application/indexes
- ../application/inputs:/app/application/inputs
- ../application/vectors:/app/application/vectors
# Host data lives under application/ (the old package directory) for this
# release, like the other compose files; it moves with the packaging work,
# together with an upgrade note. The container side is /app: the process
# runs there and resolves indexes/inputs/vectors relative to it (earlier
# versions of this file mounted /app/application/..., which nothing wrote to).
- ../application/indexes:/app/indexes
- ../application/inputs:/app/inputs
- ../application/vectors:/app/vectors
depends_on:
redis:
condition: service_started
@@ -38,14 +60,16 @@ services:
worker:
build:
context: ../application
context: ..
dockerfile: docsgpt/Dockerfile
args:
EXTRAS: ${EXTRAS:-}
INSTALL_DOCLING: ${INSTALL_DOCLING:-false}
# Off by default; deployments running OCR_ENABLED=true with tesseract
# must set INSTALL_TESSERACT=true before rebuilding (see docker-compose.yaml).
INSTALL_TESSERACT: ${INSTALL_TESSERACT:-false}
# `parsing` queue carries read_document/parse_document; required for its await to resolve.
command: celery -A application.app.celery worker -l INFO -Q docsgpt,parsing,embeddings
command: celery -A docsgpt.app.celery worker -l INFO -Q docsgpt,parsing,embeddings
env_file:
- ../.env
environment:
+29 -5
View File
@@ -1,12 +1,30 @@
# Pre-built images from Docker Hub (mirrored at ghcr.io/arc53).
# DOCSGPT_IMAGE_TAG develop (default, follows main) or a release, e.g. 0.20.0
# DOCSGPT_IMAGE_VARIANT empty (default, slim) or -docling: docling parser engine,
# its models, and tesseract baked in (OCR-ready)
# Set them in ../.env or the shell. deployment/docker-compose-standalone.yaml is
# the same stack without a git checkout.
name: docsgpt-oss
services:
frontend:
image: arc53/docsgpt-fe:develop
image: arc53/docsgpt-fe:${DOCSGPT_IMAGE_TAG:-develop}
environment:
# Every VITE_* the app reads. A bare name is passed through only when it is
# set in the shell or the --env-file, so an unset one does not reach the
# container as an empty string and override the image's own default.
- VITE_API_HOST=http://localhost:7091
- VITE_API_STREAMING=${VITE_API_STREAMING:-true}
- VITE_GOOGLE_CLIENT_ID=${VITE_GOOGLE_CLIENT_ID:-}
- VITE_BASE_URL
- VITE_GOOGLE_CLIENT_ID
- VITE_GOOGLE_PICKER_API_KEY
- VITE_SHARE_POINT_CLIENT_ID
- VITE_CONFLUENCE_CLIENT_ID
- VITE_NOTIFICATION_TEXT
- VITE_NOTIFICATION_LINK
- VITE_ENABLE_VOICE_INPUT
- VITE_DISABLE_SOURCE_FE
- VITE_USE_V
ports:
- "5173:5173"
depends_on:
@@ -15,7 +33,7 @@ services:
backend:
user: root
image: arc53/docsgpt:develop
image: arc53/docsgpt:${DOCSGPT_IMAGE_TAG:-develop}${DOCSGPT_IMAGE_VARIANT:-}
env_file:
- ../.env
environment:
@@ -26,6 +44,9 @@ services:
ports:
- "7091:7091"
volumes:
# Host data stays under application/ (the old package directory) for this
# release so an upgraded checkout keeps its indexes; it moves with the
# packaging work, together with an upgrade note.
- ../application/indexes:/app/indexes
- ../application/inputs:/app/inputs
- ../application/vectors:/app/vectors
@@ -38,9 +59,9 @@ services:
worker:
user: root
image: arc53/docsgpt:develop
image: arc53/docsgpt:${DOCSGPT_IMAGE_TAG:-develop}${DOCSGPT_IMAGE_VARIANT:-}
# `parsing` queue carries read_document/parse_document; required for its await to resolve.
command: celery -A application.app.celery worker -l INFO -B -Q docsgpt,parsing,embeddings
command: celery -A docsgpt.app.celery worker -l INFO -B -Q docsgpt,parsing,embeddings
env_file:
- ../.env
environment:
@@ -50,6 +71,9 @@ services:
- CACHE_REDIS_URL=redis://redis:6379/2
- POSTGRES_URI=postgresql://docsgpt:docsgpt@postgres:5432/docsgpt
volumes:
# Host data stays under application/ (the old package directory) for this
# release so an upgraded checkout keeps its indexes; it moves with the
# packaging work, together with an upgrade note.
- ../application/indexes:/app/indexes
- ../application/inputs:/app/inputs
- ../application/vectors:/app/vectors
+130
View File
@@ -0,0 +1,130 @@
# DocsGPT from pre-built images, with no git checkout.
#
# curl -fsSLO https://raw.githubusercontent.com/arc53/DocsGPT/main/deployment/docker-compose-standalone.yaml
# printf 'LLM_PROVIDER=docsgpt\nVITE_API_STREAMING=true\nINTERNAL_KEY=%s\n' "$(openssl rand -hex 16)" > .env
# docker compose -f docker-compose-standalone.yaml up -d
#
# INTERNAL_KEY is the shared secret the worker uses to hand finished indexes to
# the API; without it every ingest fails with a 401 (setup.sh generates one).
# open http://localhost:5173
#
# Every release also attaches this file as an asset. Settings come from .env
# next to this file (any DocsGPT setting; the compose-internal service URLs
# below take precedence). Data lives in named volumes, so `docker compose
# down` keeps it and `docker compose down -v` removes it.
#
# DOCSGPT_IMAGE_TAG release to run, e.g. 0.20.0 (default: latest release);
# develop follows the main branch
# DOCSGPT_IMAGE_VARIANT empty (slim, default) or -docling: docling parser
# engine, its models, and tesseract baked in (OCR-ready)
# EMBEDDINGS_NAME defaults to granite here (this stack always starts on
# fresh volumes, so there is no older index to keep
# compatible); the code default stays mpnet for upgrades.
name: docsgpt
services:
frontend:
image: arc53/docsgpt-fe:${DOCSGPT_IMAGE_TAG:-latest}
env_file:
- path: .env
required: false
environment:
# Every VITE_* the app reads. A bare name is passed through only when it is
# set in the shell or the --env-file, so an unset one does not reach the
# container as an empty string and override the image's own default.
- VITE_API_HOST=${VITE_API_HOST:-http://localhost:7091}
- VITE_API_STREAMING=${VITE_API_STREAMING:-true}
- VITE_BASE_URL
- VITE_GOOGLE_CLIENT_ID
- VITE_GOOGLE_PICKER_API_KEY
- VITE_SHARE_POINT_CLIENT_ID
- VITE_CONFLUENCE_CLIENT_ID
- VITE_NOTIFICATION_TEXT
- VITE_NOTIFICATION_LINK
- VITE_ENABLE_VOICE_INPUT
- VITE_DISABLE_SOURCE_FE
- VITE_USE_V
ports:
- "5173:5173"
depends_on:
- backend
backend:
image: arc53/docsgpt:${DOCSGPT_IMAGE_TAG:-latest}${DOCSGPT_IMAGE_VARIANT:-}
# Same as docker-compose-hub.yaml: the data volumes are written by root so
# any image tag works, including releases that predate the appuser-owned
# /app/inputs, /app/indexes and /app/vectors directories.
user: root
env_file:
- path: .env
required: false
environment:
- CELERY_BROKER_URL=redis://redis:6379/0
- CELERY_RESULT_BACKEND=redis://redis:6379/1
- CACHE_REDIS_URL=redis://redis:6379/2
- POSTGRES_URI=postgresql://docsgpt:docsgpt@postgres:5432/docsgpt
- EMBEDDINGS_NAME=${EMBEDDINGS_NAME:-ibm-granite/granite-embedding-311m-multilingual-r2}
ports:
- "7091:7091"
volumes:
- indexes:/app/indexes
- inputs:/app/inputs
- vectors:/app/vectors
depends_on:
redis:
condition: service_started
postgres:
condition: service_healthy
restart: unless-stopped
worker:
image: arc53/docsgpt:${DOCSGPT_IMAGE_TAG:-latest}${DOCSGPT_IMAGE_VARIANT:-}
user: root
# Consumes the default queue plus `parsing` (read_document) and `embeddings`
# (query embedding); without the latter every search times out.
command: celery -A docsgpt.app.celery worker -l INFO -B -Q docsgpt,parsing,embeddings
env_file:
- path: .env
required: false
environment:
- CELERY_BROKER_URL=redis://redis:6379/0
- CELERY_RESULT_BACKEND=redis://redis:6379/1
- CACHE_REDIS_URL=redis://redis:6379/2
- POSTGRES_URI=postgresql://docsgpt:docsgpt@postgres:5432/docsgpt
- API_URL=http://backend:7091
- EMBEDDINGS_NAME=${EMBEDDINGS_NAME:-ibm-granite/granite-embedding-311m-multilingual-r2}
volumes:
- indexes:/app/indexes
- inputs:/app/inputs
- vectors:/app/vectors
depends_on:
redis:
condition: service_started
postgres:
condition: service_healthy
restart: unless-stopped
redis:
image: redis:6-alpine
restart: unless-stopped
postgres:
image: postgres:16-alpine
environment:
- POSTGRES_USER=docsgpt
- POSTGRES_PASSWORD=docsgpt
- POSTGRES_DB=docsgpt
volumes:
- postgres_data:/var/lib/postgresql/data
healthcheck:
test: ["CMD-SHELL", "pg_isready -U docsgpt -d docsgpt"]
interval: 5s
timeout: 5s
retries: 10
restart: unless-stopped
volumes:
indexes:
inputs:
vectors:
postgres_data:
+35 -8
View File
@@ -1,13 +1,29 @@
name: docsgpt-oss
services:
frontend:
build: ../frontend
build:
context: ../frontend
# Vite dev server with hot reload over the bind mount below. The default
# target (what Docker Hub publishes) is a static build behind nginx.
target: dev
volumes:
- ../frontend/src:/app/src
environment:
# Every VITE_* the app reads. A bare name is passed through only when it is
# set in the shell or the --env-file, so an unset one does not reach the
# container as an empty string and override the image's own default.
- VITE_API_HOST=http://localhost:7091
- VITE_API_STREAMING=$VITE_API_STREAMING
- VITE_GOOGLE_CLIENT_ID=$VITE_GOOGLE_CLIENT_ID
- VITE_API_STREAMING=${VITE_API_STREAMING:-true}
- VITE_BASE_URL
- VITE_GOOGLE_CLIENT_ID
- VITE_GOOGLE_PICKER_API_KEY
- VITE_SHARE_POINT_CLIENT_ID
- VITE_CONFLUENCE_CLIENT_ID
- VITE_NOTIFICATION_TEXT
- VITE_NOTIFICATION_LINK
- VITE_ENABLE_VOICE_INPUT
- VITE_DISABLE_SOURCE_FE
- VITE_USE_V
ports:
- "5173:5173"
depends_on:
@@ -16,10 +32,13 @@ services:
backend:
user: root
build:
context: ../application
context: ..
dockerfile: docsgpt/Dockerfile
args:
# Bake the optional docling engine (layout-model OCR backend, structured
# output) into the image: set INSTALL_DOCLING=true in ../.env or the shell.
# Optional extras to bake in (comma-separated): docling, milvus. The
# docling extra brings the layout-model parser/OCR backend and its
# models. INSTALL_DOCLING=true is the older spelling of EXTRAS=docling.
EXTRAS: ${EXTRAS:-}
INSTALL_DOCLING: ${INSTALL_DOCLING:-false}
# Bake the tesseract binary behind OCR_ENABLED=true (~35 MB): set
# INSTALL_TESSERACT=true in ../.env or the shell (setup.sh does this
@@ -39,6 +58,9 @@ services:
ports:
- "7091:7091"
volumes:
# Host data stays under application/ (the old package directory) for this
# release so an upgraded checkout keeps its indexes; it moves with the
# packaging work, together with an upgrade note.
- ../application/indexes:/app/indexes
- ../application/inputs:/app/inputs
- ../application/vectors:/app/vectors
@@ -51,8 +73,10 @@ services:
worker:
user: root
build:
context: ../application
context: ..
dockerfile: docsgpt/Dockerfile
args:
EXTRAS: ${EXTRAS:-}
INSTALL_DOCLING: ${INSTALL_DOCLING:-false}
INSTALL_TESSERACT: ${INSTALL_TESSERACT:-false}
# Consumes the default queue AND the dedicated `parsing` (read_document /
@@ -61,7 +85,7 @@ services:
# fails after EMBEDDINGS_DELEGATE_TIMEOUT, because EMBEDDINGS_DELEGATE_TO_WORKER
# is on by default. For heavy/OCR parsing run a separate worker with `-Q parsing`;
# to keep query latency off the ingest pool, another with `-Q embeddings`.
command: celery -A application.app.celery worker -l INFO -B -Q docsgpt,parsing,embeddings
command: celery -A docsgpt.app.celery worker -l INFO -B -Q docsgpt,parsing,embeddings
env_file:
- ../.env
environment:
@@ -72,6 +96,9 @@ services:
- CACHE_REDIS_URL=redis://redis:6379/2
- POSTGRES_URI=postgresql://docsgpt:docsgpt@postgres:5432/docsgpt
volumes:
# Host data stays under application/ (the old package directory) for this
# release so an upgraded checkout keeps its indexes; it moves with the
# packaging work, together with an upgrade note.
- ../application/indexes:/app/indexes
- ../application/inputs:/app/inputs
- ../application/vectors:/app/vectors
@@ -42,7 +42,7 @@ spec:
name: docsgpt-secrets
env:
- name: FLASK_APP
value: "application/app.py"
value: "docsgpt/app.py"
- name: DEPLOYMENT_TYPE
value: "cloud"
- name: POSTGRES_URI
@@ -87,7 +87,7 @@ spec:
image: arc53/docsgpt
# `parsing` queue carries read_document/parse_document; required for its await to resolve.
# For heavy/OCR parsing, run a separate deployment with `-Q parsing` (and GPU env).
command: ["celery", "-A", "application.app.celery", "worker", "-l", "INFO", "-n", "worker.%h", "-Q", "docsgpt,parsing,embeddings"]
command: ["celery", "-A", "docsgpt.app.celery", "worker", "-l", "INFO", "-n", "worker.%h", "-Q", "docsgpt,parsing,embeddings"]
resources:
limits:
memory: "4Gi"
+1 -1
View File
@@ -35,7 +35,7 @@ spec:
name: docsgpt-secrets
env:
- name: FLASK_APP
value: "application/app.py"
value: "docsgpt/app.py"
resources:
limits:
memory: "1Gi"
+3 -3
View File
@@ -190,7 +190,7 @@ Document reading no longer runs in this sandbox. The `read_document` tool and th
workflow native-file extract branch enqueue a `parse_document` Celery task that
parses the document **in the backend** (the `DOC_PARSER_ENGINE` parser — anydoc
by default; Docling only when the optional
`application/requirements-docling.txt` extra is installed) and awaits the
`docsgpt/requirements-docling.txt` extra is installed) and awaits the
result. The task is routed to a
dedicated **`parsing` queue** (`settings.DOCUMENT_PARSE_QUEUE`, default
`"parsing"`) so a parse enqueued from inside a Celery worker (headless/scheduled
@@ -199,7 +199,7 @@ agent) is served by a separate worker and never self-deadlocks the awaiting one.
Run a dedicated parsing worker that consumes the `parsing` queue:
```bash
celery -A application.app.celery worker -Q parsing -l INFO
celery -A docsgpt.app.celery worker -Q parsing -l INFO
```
It takes its own env, so parse-heavy work runs on a separate, optionally larger
@@ -214,7 +214,7 @@ and leaves this worker light.
worker must also consume `parsing`, or the tool's await never resolves:
```bash
celery -A application.app.celery worker -Q docsgpt,parsing,embeddings -l INFO
celery -A docsgpt.app.celery worker -Q docsgpt,parsing,embeddings -l INFO
```
Tuning settings: `DOCUMENT_PARSE_TIMEOUT` (seconds the tool awaits before
+3 -3
View File
@@ -44,7 +44,7 @@ The main set of instructions or system [prompt](/Guides/Customising-prompts) tha
## Understanding Agent Types
DocsGPT supports several agent types, each with a distinct way of processing information. The code for these can be found in the `application/agents/` directory.
DocsGPT supports several agent types, each with a distinct way of processing information. The code for these can be found in the `docsgpt/agents/` directory.
### 1. Classic Agent
@@ -117,8 +117,8 @@ Once an agent is created, you can:
You can bootstrap a fresh DocsGPT deployment with a curated set of agents by seeding them directly into the user-data store (Postgres).
1. **Customize the configuration** – edit `application/seed/config/premade_agents.yaml` (or copy from `application/seed/config/agents_template.yaml`) to describe the agents you want to provision. Each entry lets you define prompts, tools, and optional data sources.
1. **Customize the configuration** – edit `docsgpt/seed/config/premade_agents.yaml` (or copy from `docsgpt/seed/config/agents_template.yaml`) to describe the agents you want to provision. Each entry lets you define prompts, tools, and optional data sources.
2. **Ensure dependencies are running** – Postgres must be reachable using `POSTGRES_URI` from `.env` (schema applied via `python scripts/db/init_postgres.py`), and a Celery worker should be available if any agent sources need to be ingested via `ingest_remote`.
3. **Execute the seeder** – run `python -m application.seed.commands init`. Add `--force` when you need to reseed an existing environment.
3. **Execute the seeder** – run `python -m docsgpt.seed.commands init`. Add `--force` when you need to reseed an existing environment.
The seeder keeps templates under the `system` user so they appear in the UI for anyone to clone or customize. Environment variable placeholders such as `${MY_TOKEN}` inside tool configs are resolved during the seeding process.
+1 -1
View File
@@ -70,7 +70,7 @@ GET /api/messages/<message_id>/events
This replays the message's events past your last-seen sequence number and tails the rest live. It is backed by the Postgres `message_events` journal (retained for `MESSAGE_EVENTS_RETENTION_DAYS`, default 14).
<Callout type="warning" emoji="⚠️">
The chat reconnect endpoint is a native-async route served by the ASGI entrypoint. Under a plain `flask run` dev server it returns `404`; run the backend via the ASGI app (`uvicorn application.asgi:asgi_app`) or the production gunicorn uvicorn worker to use it. See the [Development Environment](/Deploying/Development-Environment) guide.
The chat reconnect endpoint is a native-async route served by the ASGI entrypoint. Under a plain `flask run` dev server it returns `404`; run the backend via the ASGI app (`uvicorn docsgpt.asgi:asgi_app`) or the production gunicorn uvicorn worker to use it. See the [Development Environment](/Deploying/Development-Environment) guide.
</Callout>
## Settings
@@ -53,7 +53,7 @@ To run the DocsGPT backend locally, you'll need to set up a Python environment a
* **Option 1: Using a `.env` file (Recommended):**
* If you haven't already, create a file named `.env` in the **root directory** of your DocsGPT project.
* Modify the `.env` file to adjust settings as needed. You can find a comprehensive list of configurable options in [`application/core/settings.py`](https://github.com/arc53/DocsGPT/blob/main/application/core/settings.py).
* Modify the `.env` file to adjust settings as needed. You can find a comprehensive list of configurable options in [`docsgpt/core/settings.py`](https://github.com/arc53/DocsGPT/blob/main/docsgpt/core/settings.py).
* **Option 2: Exporting Environment Variables:**
* Alternatively, you can export environment variables directly in your terminal. However, using a `.env` file is generally more organized for development.
@@ -83,7 +83,7 @@ To run the DocsGPT backend locally, you'll need to set up a Python environment a
For an offline or air-gapped machine, fetch it ahead of time instead:
```bash
python -m application.scripts.prefetch_models
python -m docsgpt.scripts.prefetch_models
```
4. **Install Backend Dependencies:**
@@ -91,40 +91,56 @@ To run the DocsGPT backend locally, you'll need to set up a Python environment a
Navigate to the root of your DocsGPT repository and install the required Python packages:
```bash
pip install -r application/requirements.txt
pip install -r docsgpt/requirements.txt
```
Optionally add the docling parser engine (OCR, `structured` output for
`read_document`; the default anydoc engine does not need it):
Dependencies are declared in `pyproject.toml` and locked in `uv.lock`; the
`requirements*.txt` files are exported from that lock, so with
[uv](https://docs.astral.sh/uv/) installed `uv sync` sets up the same
environment (plus the test tools) in one step.
Optional extras are not installed by default. Add them when you need the
feature (each file is the core set plus the extra):
```bash
pip install -r application/requirements-docling.txt
pip install -r docsgpt/requirements-docling.txt # docling parser engine: OCR backend, read_document structured output
pip install -r docsgpt/requirements-milvus.txt # VECTOR_STORE=milvus
# or with uv: uv sync --extra docling --extra milvus
```
The docling file adds the PyTorch CPU index; pip handles that as-is, while
`uv pip install -r` needs `UV_INDEX_STRATEGY=unsafe-best-match` (or use
`uv sync --extra docling`, which reads the lock).
A feature whose extra is missing fails with the exact command to run.
After changing `pyproject.toml`, run `uv lock` and
`bash scripts/export_requirements.sh` so the exported files stay in sync
(CI checks this).
5. **Run the Backend:**
For local development, run the ASGI composition under uvicorn. It serves the **whole** application, hot-reloads on source changes, and matches the production runtime:
```bash
uvicorn application.asgi:asgi_app --host 0.0.0.0 --port 7091 --reload
uvicorn docsgpt.asgi:asgi_app --host 0.0.0.0 --port 7091 --reload
```
This makes the backend accessible on `http://localhost:7091`. Production uses `gunicorn -k uvicorn_worker.UvicornWorker` against the same `application.asgi:asgi_app` target.
This makes the backend accessible on `http://localhost:7091`. Production uses `gunicorn -k uvicorn_worker.UvicornWorker` against the same `docsgpt.asgi:asgi_app` target.
A plain Flask run is a faster inner loop (quick startup, the Werkzeug interactive debugger):
```bash
flask --app application/app.py run --host=0.0.0.0 --port=7091
flask --app docsgpt/app.py run --host=0.0.0.0 --port=7091
```
But it serves **only** the WSGI Flask app and omits the routes mounted on the ASGI shell in `application/asgi.py`: the `/mcp` FastMCP endpoint and the native-async SSE reconnect reader `GET /api/messages/<id>/events`. Under `flask run` those paths return 404 — chat still works (`POST /stream` is a Flask route), but a stream interrupted by a disconnect won't auto-resume on reconnect. Use `flask run` only when you don't need those routes.
But it serves **only** the WSGI Flask app and omits the routes mounted on the ASGI shell in `docsgpt/asgi.py`: the `/mcp` FastMCP endpoint and the native-async SSE reconnect reader `GET /api/messages/<id>/events`. Under `flask run` those paths return 404 — chat still works (`POST /stream` is a Flask route), but a stream interrupted by a disconnect won't auto-resume on reconnect. Use `flask run` only when you don't need those routes.
6. **Start the Celery Worker:**
Open a new terminal window (and activate your virtual environment if you used one). Start the Celery worker to handle background tasks:
```bash
celery -A application.app.celery worker -l INFO
celery -A docsgpt.app.celery worker -l INFO
```
This command will start the Celery worker, which processes tasks such as document parsing and vector embedding.
@@ -132,7 +148,7 @@ To run the DocsGPT backend locally, you'll need to set up a Python environment a
**macOS note:** Due to a threading issue, start Celery with the solo pool:
```bash
python -m celery -A application.app.celery worker -l INFO --pool=solo
python -m celery -A docsgpt.app.celery worker -l INFO --pool=solo
```
**Running in Debugger (VSCode):**
+71 -11
View File
@@ -17,9 +17,62 @@ Docker is the recommended method for deploying DocsGPT, providing a consistent a
**Important Note for Windows Users:** Docker Desktop on Windows generally requires the WSL 2 backend to function correctly, especially when using features like host networking which are utilized in DocsGPT's Docker Compose setup. Ensure WSL 2 is enabled and configured in Docker Desktop settings.
## Quickest Setup: Using DocsGPT Public API
## Quickest Setup: Pre-built Images, No Checkout
The fastest way to try out DocsGPT is by using the public API endpoint. This requires minimal configuration and no local LLM setup.
Every release publishes ready-to-run images to Docker Hub (`arc53/docsgpt`,
`arc53/docsgpt-fe`) and GitHub Container Registry (`ghcr.io/arc53/docsgpt`,
`ghcr.io/arc53/docsgpt-fe`) for `linux/amd64` and `linux/arm64`. The images
contain everything the default configuration needs (embedding models,
tokenizers, tiktoken's encoding), so a fresh container makes no downloads on
first use. You do not need the source tree to run them:
1. **Download the standalone Compose file** (also attached to every
[release](https://github.com/arc53/DocsGPT/releases)):
```bash
mkdir docsgpt && cd docsgpt
curl -fsSLO https://raw.githubusercontent.com/arc53/DocsGPT/main/deployment/docker-compose-standalone.yaml
```
2. **Create a `.env` next to it** with your settings, for example the public API:
```bash
printf 'LLM_PROVIDER=docsgpt\nVITE_API_STREAMING=true\nINTERNAL_KEY=%s\n' "$(openssl rand -hex 16)" > .env
```
`INTERNAL_KEY` is the secret the worker uses to hand finished indexes to
the API; without it every upload fails with a 401. `setup.sh` generates
one for you, a hand-written `.env` has to include it. This stack runs the
granite embedding model unless `.env` sets `EMBEDDINGS_NAME`; both granite
and mpnet are baked into the image.
3. **Start it:**
```bash
docker compose -f docker-compose-standalone.yaml up -d
```
Then open [http://localhost:5173/](http://localhost:5173/). Data lives in
named Docker volumes; `docker compose -f docker-compose-standalone.yaml down`
keeps it and `down -v` removes it.
**Tags and variants.** `DOCSGPT_IMAGE_TAG` picks the version: a release such
as `0.20.0`, `latest` (the newest release, the default) or `develop` (follows
the `main` branch). `DOCSGPT_IMAGE_VARIANT` picks the flavour: empty for the
slim default image, or `-docling` for the image with the docling parser
engine, its models and tesseract baked in (needed for OCR of scanned
documents, see the [OCR guide](/Guides/ocr)). Both are read from `.env` or
the shell, e.g. `DOCSGPT_IMAGE_TAG=0.20.0 DOCSGPT_IMAGE_VARIANT=-docling`.
The same two variables drive `deployment/docker-compose-hub.yaml` in a
checkout.
## Using the Source Checkout
With a clone of the repository, `deployment/docker-compose-hub.yaml` runs the
same pre-built images while keeping your data in `application/indexes`,
`application/inputs` and `application/vectors`, and `deployment/docker-compose.yaml`
builds the images from your working tree (for local changes, or a build with
extra packages: `EXTRAS=docling` in `.env`).
1. **Clone the DocsGPT Repository (if you haven't already):**
@@ -39,19 +92,26 @@ The fastest way to try out DocsGPT is by using the public API endpoint. This req
```
LLM_PROVIDER=docsgpt
VITE_API_STREAMING=true
INTERNAL_KEY=<any random string, e.g. openssl rand -hex 16>
EMBEDDINGS_NAME=ibm-granite/granite-embedding-311m-multilingual-r2
```
This minimal configuration tells DocsGPT to use the public API. For more advanced settings and other LLM options, refer to the [DocsGPT Settings Guide](/Deploying/DocsGPT-Settings).
This minimal configuration tells DocsGPT to use the public API. The
`EMBEDDINGS_NAME` line is what `setup.sh` writes for a new install; without
it the code falls back to mpnet, the model earlier releases indexed with,
so that an upgraded deployment keeps its existing index working. For more advanced settings and other LLM options, refer to the [DocsGPT Settings Guide](/Deploying/DocsGPT-Settings).
4. **Launch DocsGPT with Docker Compose:**
Navigate to the root directory of the DocsGPT repository in your terminal and run:
```bash
docker compose --env-file .env -f deployment/docker-compose.yaml up -d
docker compose --env-file .env -f deployment/docker-compose-hub.yaml up -d
```
The `-d` flag runs Docker Compose in detached mode (in the background).
To build the images from your working tree instead of pulling them, use
`deployment/docker-compose.yaml` with `up --build -d`.
5. **Access DocsGPT in your browser:**
@@ -62,7 +122,7 @@ The fastest way to try out DocsGPT is by using the public API endpoint. This req
To stop the application, navigate to the same directory in your terminal and run:
```bash
docker compose -f deployment/docker-compose.yaml down
docker compose -f deployment/docker-compose-hub.yaml down
```
## Optional Ollama Setup (Local Models)
@@ -84,11 +144,11 @@ There are two Ollama optional files:
**CPU:**
```bash
docker compose --env-file .env -f deployment/docker-compose.yaml -f deployment/optional/docker-compose.optional.ollama-cpu.yaml up -d
docker compose --env-file .env -f deployment/docker-compose-hub.yaml -f deployment/optional/docker-compose.optional.ollama-cpu.yaml up -d
```
**GPU:**
```bash
docker compose --env-file .env -f deployment/docker-compose.yaml -f deployment/optional/docker-compose.optional.ollama-gpu.yaml up -d
docker compose --env-file .env -f deployment/docker-compose-hub.yaml -f deployment/optional/docker-compose.optional.ollama-gpu.yaml up -d
```
3. **Pull the Ollama Model:**
@@ -96,11 +156,11 @@ There are two Ollama optional files:
**Crucially, after launching with Ollama, you need to pull the desired model into the Ollama container.** Find the `LLM_NAME` you configured in your `.env` file (e.g., `llama3.2:1b`). Then execute the following command to pull the model *inside* the running Ollama container:
```bash
docker compose -f deployment/docker-compose.yaml -f deployment/optional/docker-compose.optional.ollama-cpu.yaml exec -it ollama ollama pull <LLM_NAME>
docker compose -f deployment/docker-compose-hub.yaml -f deployment/optional/docker-compose.optional.ollama-cpu.yaml exec -it ollama ollama pull <LLM_NAME>
```
or (for GPU):
```bash
docker compose -f deployment/docker-compose.yaml -f deployment/optional/docker-compose.optional.ollama-gpu.yaml exec -it ollama ollama pull <LLM_NAME>
docker compose -f deployment/docker-compose-hub.yaml -f deployment/optional/docker-compose.optional.ollama-gpu.yaml exec -it ollama ollama pull <LLM_NAME>
```
Replace `<LLM_NAME>` with the actual model name from your `.env` file.
@@ -113,12 +173,12 @@ There are two Ollama optional files:
To stop a DocsGPT setup launched with Ollama optional files, use `docker compose down` and include all the compose files used during the `up` command:
```bash
docker compose -f deployment/docker-compose.yaml -f deployment/optional/docker-compose.optional.ollama-cpu.yaml down
docker compose -f deployment/docker-compose-hub.yaml -f deployment/optional/docker-compose.optional.ollama-cpu.yaml down
```
or
```bash
docker compose -f deployment/docker-compose.yaml -f deployment/optional/docker-compose.optional.ollama-gpu.yaml down
docker compose -f deployment/docker-compose-hub.yaml -f deployment/optional/docker-compose.optional.ollama-gpu.yaml down
```
**Important for GPU Usage:**
+10 -9
View File
@@ -29,11 +29,11 @@ LLM_NAME=gpt-4o
### 2. Configuration via `settings.py` file (Advanced)
For more advanced configurations or if you prefer to manage settings directly in code, you can modify the `settings.py` file. This file is located in the `application/core` directory of your DocsGPT project.
For more advanced configurations or if you prefer to manage settings directly in code, you can modify the `settings.py` file. This file is located in the `docsgpt/core` directory of your DocsGPT project.
While modifying `settings.py` offers more flexibility, it's generally recommended to use the `.env` file for basic settings and reserve `settings.py` for more complex adjustments or when you need to configure settings programmatically.
**Location of `settings.py`:** `application/core/settings.py`
**Location of `settings.py`:** `docsgpt/core/settings.py`
## Basic Settings Explained
@@ -61,7 +61,7 @@ Here are some of the most fundamental settings you'll likely want to configure:
- **Default value:** leave it unset and DocsGPT picks for you at first boot, recording the choice so it never changes underneath you: a fresh install is pinned to `ibm-granite/granite-embedding-311m-multilingual-r2` (multilingual, 32k context, same 768 dimensions), and an install that already has sources is pinned to `huggingface_sentence-transformers/all-mpnet-base-v2` so its index stays readable. Setting it here overrides that pin.
- **Other options:** Any FastEmbed built-in model, or any Hugging Face repository shipping an ONNX export. See [Embeddings](/Models/embeddings).
- **Changing it on an existing index requires re-embedding** — same-width models swap without any error and silently degrade retrieval. Run `python -m application.scripts.reembed`.
- **Changing it on an existing index requires re-embedding** — same-width models swap without any error and silently degrade retrieval. Run `python -m docsgpt.scripts.reembed`.
- **`API_KEY`**: Required for most cloud-based LLM providers. This is your authentication key to access the LLM provider's API. You'll need to obtain this key from your chosen provider's platform.
@@ -133,7 +133,7 @@ models:
After restart, those models appear in `/api/models` and are selectable
in the UI. A working template lives at
`application/core/models/examples/mistral.yaml.example`.
`docsgpt/core/models/examples/mistral.yaml.example`.
**What you can do:**
@@ -148,8 +148,8 @@ in the UI. A working template lives at
**What you cannot do via `MODELS_CONFIG_DIR`:** add a brand-new
non-OpenAI provider. That requires a Python plugin under
`application/llm/providers/`. See
`application/core/models/README.md` for the full schema reference.
`docsgpt/llm/providers/`. See
`docsgpt/core/models/README.md` for the full schema reference.
### Docker
@@ -207,7 +207,7 @@ for the engines and flows.
| Setting | Default | Description |
| --- | --- | --- |
| `DOC_PARSER_ENGINE` | `anydoc` | Parser engine: `anydoc` (fast Rust converter, no ML models) or `docling` (layout/table models, structured output). Files anydoc cannot read fall back to Docling when installed. Docling is an optional extra: `pip install -r application/requirements-docling.txt`, or Docker builds with `--build-arg INSTALL_DOCLING=true`. |
| `DOC_PARSER_ENGINE` | `anydoc` | Parser engine: `anydoc` (fast Rust converter, no ML models) or `docling` (layout/table models, structured output). Files anydoc cannot read fall back to Docling when installed. Docling is an optional extra: `pip install -r docsgpt/requirements-docling.txt`, or Docker builds with `--build-arg INSTALL_DOCLING=true`. |
| `OCR_ENABLED` | `false` | OCR for source ingestion. Alias: `DOCLING_OCR_ENABLED`. |
| `OCR_ATTACHMENTS_ENABLED` | `false` | OCR for chat attachments. Alias: `DOCLING_OCR_ATTACHMENTS_ENABLED`. |
| `OCR_BACKEND` | `auto` | Who performs OCR: `auto` (Docling when installed, else native), `native` (pypdfium2 + Pillow rendering into tesseract or DeepSeek-OCR; no docling needed), or `docling` (layout-model hybrid OCR). See the [OCR guide](/Guides/ocr#ocr-backends). |
@@ -469,8 +469,9 @@ See [Embeddings](/Models/embeddings) for full guidance.
| Setting | Default | Description |
| --- | --- | --- |
| `EMBEDDINGS_NAME` | `huggingface_sentence-transformers/all-mpnet-base-v2` | The embedding model. New installs use `ibm-granite/granite-embedding-311m-multilingual-r2`. Changing it on a populated index requires `application.scripts.reembed`. |
| `EMBEDDINGS_NAME` | `huggingface_sentence-transformers/all-mpnet-base-v2` | The embedding model. New installs use `ibm-granite/granite-embedding-311m-multilingual-r2`. Changing it on a populated index requires `docsgpt.scripts.reembed`. |
| `EMBEDDINGS_BASE_URL` | unset | Base URL of a remote OpenAI-compatible embeddings server. Setting it routes all embedding calls there. |
| `EMBEDDINGS_THREADS` | unset (Docker image: `4`) | Threads one local FastEmbed/onnxruntime session may use. onnxruntime otherwise sizes its pool to the host's core count, which a CPU-limited container still reports, so the image pins it like `OMP_NUM_THREADS`. Raise it on a large dedicated worker. |
| `EMBEDDINGS_KEY` | unset | Optional bearer token for the remote embeddings server. |
| `EMBEDDINGS_MAX_INPUT_TOKENS` | unset | Truncate each remote embedding input to N tokens (guards servers that reject oversized inputs). |
| `EMBEDDINGS_DELEGATE_TO_WORKER` | `true` | Embed queries on the Celery worker instead of loading a model in the API. Requires a worker consuming `EMBEDDINGS_QUEUE`; set `false` to run the API standalone. Ignored when `EMBEDDINGS_BASE_URL` is set. |
@@ -549,4 +550,4 @@ These are just the basic settings to get you started. The `settings.py` file con
- Cache settings (`CACHE_REDIS_URL`)
- And many more!
For a complete list of available settings and their descriptions, refer to the `settings.py` file in `application/core`. Remember to restart your Docker containers after making changes to your `.env` file or `settings.py` for the changes to take effect.
For a complete list of available settings and their descriptions, refer to the `settings.py` file in `docsgpt/core`. Remember to restart your Docker containers after making changes to your `.env` file or `settings.py` for the changes to take effect.
+7 -7
View File
@@ -8,7 +8,7 @@ import { Callout } from 'nextra/components'
# Observability
DocsGPT bundles the OpenTelemetry SDK and auto-instrumentation packages
in `application/requirements.txt` — they install with the rest of the
in `docsgpt/requirements.txt` — they install with the rest of the
backend deps. Telemetry is **off by default**; opt in by prefixing the
launch command with `opentelemetry-instrument` and setting OTLP env
vars.
@@ -42,12 +42,12 @@ services:
backend:
command: >
opentelemetry-instrument gunicorn -w 1 -k uvicorn_worker.UvicornWorker
--bind 0.0.0.0:7091 --config application/gunicorn_conf.py
application.asgi:asgi_app
--bind 0.0.0.0:7091 --config docsgpt/gunicorn_conf.py
docsgpt.asgi:asgi_app
environment:
- OTEL_SERVICE_NAME=docsgpt-backend
worker:
command: opentelemetry-instrument celery -A application.app.celery worker -l INFO -B
command: opentelemetry-instrument celery -A docsgpt.app.celery worker -l INFO -B
environment:
- OTEL_SERVICE_NAME=docsgpt-celery-worker
```
@@ -56,14 +56,14 @@ For local dev, prepend `dotenv run --` so the `OTEL_*` vars from `.env`
reach `opentelemetry-instrument` before it boots the SDK:
```bash
dotenv run -- opentelemetry-instrument flask --app application/app.py run --port=7091
dotenv run -- opentelemetry-instrument celery -A application.app.celery worker -l INFO --pool=solo
dotenv run -- opentelemetry-instrument flask --app docsgpt/app.py run --port=7091
dotenv run -- opentelemetry-instrument celery -A docsgpt.app.celery worker -l INFO --pool=solo
```
<Callout type="info" emoji="ℹ️">
Logs are exported in-process when `OTEL_LOGS_EXPORTER=otlp` is set —
`application/core/logging_config.py` detects the flag and preserves
`docsgpt/core/logging_config.py` detects the flag and preserves
the OTEL log handler. Without it, `logging` writes only to stdout.
</Callout>
@@ -37,7 +37,7 @@ schema on first boot.
```bash
export POSTGRES_URI="postgresql://user:pass@host/docsgpt?sslmode=require"
uvicorn application.asgi:asgi_app --host 0.0.0.0 --port 7091
uvicorn docsgpt.asgi:asgi_app --host 0.0.0.0 --port 7091
```
### Bare-metal Postgres
@@ -47,7 +47,7 @@ First boot creates both the database and the schema.
```bash
export POSTGRES_URI="postgresql://postgres@localhost/docsgpt"
uvicorn application.asgi:asgi_app --host 0.0.0.0 --port 7091
uvicorn docsgpt.asgi:asgi_app --host 0.0.0.0 --port 7091
```
Prefer a dedicated non-superuser role? Create it once as superuser — the
@@ -93,7 +93,7 @@ init-container ahead of the app rollout:
```bash
python scripts/db/init_postgres.py
# equivalently:
alembic -c application/alembic.ini upgrade head
alembic -c docsgpt/alembic.ini upgrade head
```
The reasoning: the app's runtime role shouldn't carry DDL privileges,
@@ -112,7 +112,7 @@ One-shot, offline, app stopped. The app itself will create the
Postgres schema when it boots — you only need to run the data copy.
```bash
pip install -r application/requirements.txt
pip install -r docsgpt/requirements.txt
pip install 'pymongo>=4.6'
export POSTGRES_URI="postgresql://docsgpt:docsgpt@localhost:5432/docsgpt"
+2 -2
View File
@@ -359,7 +359,7 @@ Technical documentation about...
### Template Validation
Test your template syntax before saving:
```python
from application.api.answer.services.prompt_renderer import PromptRenderer
from docsgpt.api.answer.services.prompt_renderer import PromptRenderer
renderer = PromptRenderer()
is_valid = renderer.validate_template("Your prompt with {{ variables }}")
@@ -487,7 +487,7 @@ Provide detailed answers appropriate for {{ passthrough.access_level }} access l
### Render Prompt via API
```python
from application.api.answer.services.prompt_renderer import PromptRenderer
from docsgpt.api.answer.services.prompt_renderer import PromptRenderer
renderer = PromptRenderer()
rendered = renderer.render_prompt(
+1 -1
View File
@@ -19,7 +19,7 @@ The compression system operates on a "summarize and truncate" principle:
## Configuration
You can configure the compression behavior in your `.env` file or `application/core/settings.py`:
You can configure the compression behavior in your `.env` file or `docsgpt/core/settings.py`:
| Setting | Default | Description |
| :--- | :--- | :--- |
+28 -14
View File
@@ -57,18 +57,24 @@ apt-get install tesseract-ocr tesseract-ocr-eng brew install tesseract
Docker images build without it by default; opt in with the build argument:
```bash
docker build --build-arg INSTALL_TESSERACT=true ./application
docker build -f docsgpt/Dockerfile --build-arg INSTALL_TESSERACT=true .
```
`deployment/docker-compose.yaml` forwards the same switch, so setting
`INSTALL_TESSERACT=true` in `.env` (or the shell) bakes tesseract plus the
English pack into locally built backend and worker images; `setup.sh` writes
it when you answer yes to the OCR question after choosing to build images
locally. Pre-built Docker Hub images (`docker-compose-hub.yaml`) do not
include it, so `setup.sh` leaves OCR at its default (off) for them; to OCR
there, point `OCR_ENGINE=deepseek` at a DeepSeek-OCR endpoint or install
`tesseract-ocr` in a derived image. With `OCR_ENABLED=true` and no binary on
`PATH`, scanned pages fail with an install hint (text-layer documents are
locally.
With pre-built images the switch is the image variant: every tag is
published twice, slim (`arc53/docsgpt:<tag>`) and `-docling`
(`arc53/docsgpt:<tag>-docling`), and the latter bakes tesseract, the docling
engine and its models in. Set `DOCSGPT_IMAGE_VARIANT=-docling` in `.env` for
`docker-compose-hub.yaml` or `docker-compose-standalone.yaml`; `setup.sh`
writes it when you answer yes to the OCR question with Docker Hub images.
Alternatively point `OCR_ENGINE=deepseek` at a DeepSeek-OCR endpoint, which
needs no system package. With `OCR_ENABLED=true` and no binary on `PATH`,
scanned pages fail with an install hint (text-layer documents are
unaffected).
<Callout type="warning" emoji="⚠️">
@@ -88,23 +94,31 @@ docling is not part of the base install, and OCR does not need it (see
output:
```bash
pip install -r application/requirements-docling.txt
pip install -r docsgpt/requirements-docling.txt # or: uv sync --extra docling
```
Docker images build without it by default; opt in with the build argument:
That file is the core set plus the `docling` extra, exported from the same
lock. On Linux it takes torch from the CPU-only PyTorch index, so the extra
costs about 1.5 GB rather than the 2.7 GB the CUDA build of torch would; a
GPU deployment can reinstall torch from PyPI on top.
Pre-built images: use the `-docling` variant (`arc53/docsgpt:<tag>-docling`,
`DOCSGPT_IMAGE_VARIANT=-docling` in `.env`), which also bakes docling's
layout, table-structure and RapidOCR models in so the first parse does not
download them. Local builds opt in with the build argument:
```bash
docker build --build-arg INSTALL_DOCLING=true ./application
docker build -f docsgpt/Dockerfile --build-arg EXTRAS=docling .
```
`deployment/docker-compose.yaml` forwards the same switch, so setting
`INSTALL_DOCLING=true` in `.env` (or the shell) bakes docling into locally
built backend and worker images; `setup.sh` offers it as a follow-up to the
OCR question. Compose reads build arguments from the shell or from the
`EXTRAS=docling` (or the older `INSTALL_DOCLING=true`) in `.env` (or the
shell) bakes docling into locally built backend and worker images; `setup.sh`
offers it as a follow-up to the OCR question. Compose reads build arguments from the shell or from the
`.env` you pass with `--env-file .env` (not from the containers' `env_file`),
so build with `docker compose --env-file .env -f deployment/docker-compose.yaml build`
as `setup.sh` does. Pre-built Docker Hub images (`docker-compose-hub.yaml`)
never include docling; install it in a derived image instead. Either way no
as `setup.sh` does. Of the pre-built Docker Hub images only the slim default
excludes docling; the `-docling` variant ships it with its models. Either way no
code changes are needed — docling is picked up as
the fallback engine (and, under `OCR_BACKEND=auto`, as the OCR backend) as
soon as it is importable, and `DOC_PARSER_ENGINE=docling` makes it the
+2 -2
View File
@@ -33,7 +33,7 @@ DocsGPT offers direct, streamlined support for the following cloud LLM providers
| Novita AI | `novita` | (See Novita docs) |
| HuggingFace Inference API | `huggingface` | `meta-llama/Llama-3.1-8B-Instruct` |
DocsGPT also ships a **model catalog** (`application/core/models/*.yaml`) that the in-app model picker reads, so common models from these providers — including DeepSeek — appear ready to select once the matching API key is set.
DocsGPT also ships a **model catalog** (`docsgpt/core/models/*.yaml`) that the in-app model picker reads, so common models from these providers — including DeepSeek — appear ready to select once the matching API key is set.
## Connecting to OpenAI-Compatible Cloud APIs
@@ -75,4 +75,4 @@ See [App Configuration](/Deploying/DocsGPT-Settings) for the full settings refer
## Adding Support for Other Cloud Providers
If you wish to connect to a cloud provider that is not explicitly listed above or doesn't offer OpenAI API compatibility, you can extend DocsGPT to support it. Within the DocsGPT repository, navigate to the `application/llm` directory. Here, you will find Python files defining the existing LLM integrations. You can use these files as examples to create a new module for your desired cloud provider. After creating your new LLM module, you will need to register it within the `llm_creator.py` file. This process involves some coding, but it allows for virtually unlimited extensibility to connect to any cloud-based LLM service with an accessible API.
If you wish to connect to a cloud provider that is not explicitly listed above or doesn't offer OpenAI API compatibility, you can extend DocsGPT to support it. Within the DocsGPT repository, navigate to the `docsgpt/llm` directory. Here, you will find Python files defining the existing LLM integrations. You can use these files as examples to create a new module for your desired cloud provider. After creating your new LLM module, you will need to register it within the `llm_creator.py` file. This process involves some coding, but it allows for virtually unlimited extensibility to connect to any cloud-based LLM service with an accessible API.
+4 -4
View File
@@ -46,7 +46,7 @@ Models with a Dense projection layer (for example `sentence-transformers/LaBSE`)
For an offline or air-gapped install, pre-fetch the model at build or setup time:
```bash
python -m application.scripts.prefetch_models
python -m docsgpt.scripts.prefetch_models
```
## Using OpenAI Embeddings
@@ -100,7 +100,7 @@ Retrieval then depends on a worker consuming `EMBEDDINGS_QUEUE` (`embeddings` by
Sharing one worker also shares its concurrency with ingest, so a query can queue behind a long parse. Run a dedicated worker to isolate query latency:
```bash
celery -A application.app.celery worker -Q embeddings
celery -A docsgpt.app.celery worker -Q embeddings
```
Set `EMBEDDINGS_DELEGATE_TO_WORKER=false` if you run the API without a worker; it will load the model in-process instead.
@@ -139,7 +139,7 @@ The dimension check is a guard against a corrupt index, not a guarantee that a s
Switching between same-width models therefore still requires re-embedding:
```bash
python -m application.scripts.reembed
python -m docsgpt.scripts.reembed
```
Run it after changing `EMBEDDINGS_NAME` and before serving queries. See [Upgrading](/upgrading) for the granite migration specifically.
@@ -150,6 +150,6 @@ With `GRAPHRAG_ENABLED`, the script also rewrites `graph_nodes.name_embedding` o
## Adding Support for Other Embedding Models
To teach DocsGPT about a new model — so it carries a known pooling, width and context window rather than being inferred — add an `EmbeddingModel` entry to `MODELS` in `application/vectorstore/model_registry.py`. That registry is the single source of truth the local runner, the remote client, the schema bootstrap and the chunker all read.
To teach DocsGPT about a new model — so it carries a known pooling, width and context window rather than being inferred — add an `EmbeddingModel` entry to `MODELS` in `docsgpt/vectorstore/model_registry.py`. That registry is the single source of truth the local runner, the remote client, the schema bootstrap and the chunker all read.
Specifically, pay attention to the `EmbeddingsWrapper` and `EmbeddingsSingleton` classes. `EmbeddingsWrapper` provides a way to wrap different embedding model libraries into a consistent interface for DocsGPT. `EmbeddingsSingleton` manages the instantiation and retrieval of embedding model instances. By understanding these classes and the existing embedding model implementations, you can create your own custom integration for virtually any embedding model library you desire.
+1 -1
View File
@@ -86,4 +86,4 @@ You can configure multiple API keys simultaneously (e.g., both `OPENAI_API_KEY`
## Adding Support for Other Local Engines
While DocsGPT currently focuses on OpenAI API compatible local engines, you can extend its capabilities to support other local inference solutions. To do this, navigate to the `application/llm` directory in the DocsGPT repository. Examine the existing Python files for examples of LLM integrations. You can create a new module for your desired local engine, and then register it in the `llm_creator.py` file within the same directory. This allows for custom integration with a wide range of local LLM servers beyond those listed above.
While DocsGPT currently focuses on OpenAI API compatible local engines, you can extend its capabilities to support other local inference solutions. To do this, navigate to the `docsgpt/llm` directory in the DocsGPT repository. Examine the existing Python files for examples of LLM integrations. You can create a new module for your desired local engine, and then register it in the `llm_creator.py` file within the same directory. This allows for custom integration with a wide range of local LLM servers beyond those listed above.
+10 -10
View File
@@ -38,37 +38,37 @@ DocsGPT includes a suite of pre-built tools designed to expand its capabilities
},
{
title: 'Brave Search',
link: 'https://github.com/arc53/DocsGPT/blob/main/application/agents/tools/brave.py',
link: 'https://github.com/arc53/DocsGPT/blob/main/docsgpt/agents/tools/brave.py',
description: 'Enables DocsGPT to perform real-time web and image searches using the Brave Search API. Requires an API key.'
},
{
title: 'DuckDuckGo Search',
link: 'https://github.com/arc53/DocsGPT/blob/main/application/agents/tools/duckduckgo.py',
link: 'https://github.com/arc53/DocsGPT/blob/main/docsgpt/agents/tools/duckduckgo.py',
description: 'Performs web and image searches using DuckDuckGo. No API key required.'
},
{
title: 'CryptoPrice',
link: 'https://github.com/arc53/DocsGPT/blob/main/application/agents/tools/cryptoprice.py',
link: 'https://github.com/arc53/DocsGPT/blob/main/docsgpt/agents/tools/cryptoprice.py',
description: 'Fetches the current price of specified cryptocurrencies using the CryptoCompare public API.'
},
{
title: 'Ntfy',
link: 'https://github.com/arc53/DocsGPT/blob/main/application/agents/tools/ntfy.py',
link: 'https://github.com/arc53/DocsGPT/blob/main/docsgpt/agents/tools/ntfy.py',
description: 'Allows DocsGPT to send push notifications to ntfy topics on a specified server, ideal for alerts and updates.'
},
{
title: 'Telegram Bot',
link: 'https://github.com/arc53/DocsGPT/blob/main/application/agents/tools/telegram.py',
link: 'https://github.com/arc53/DocsGPT/blob/main/docsgpt/agents/tools/telegram.py',
description: 'Allows DocsGPT to send messages or images to Telegram chats via a Telegram Bot. Requires a bot token and chat ID.'
},
{
title: 'PostgreSQL Database',
link: 'https://github.com/arc53/DocsGPT/blob/main/application/agents/tools/postgres.py',
link: 'https://github.com/arc53/DocsGPT/blob/main/docsgpt/agents/tools/postgres.py',
description: 'Connects to a PostgreSQL database to execute SQL queries and retrieve schema information.'
},
{
title: 'Read Webpage (browser)',
link: 'https://github.com/arc53/DocsGPT/blob/main/application/agents/tools/read_webpage.py',
link: 'https://github.com/arc53/DocsGPT/blob/main/docsgpt/agents/tools/read_webpage.py',
description: 'Fetches the HTML content of a URL and converts it to Markdown for the agent to read.'
},
{
@@ -83,17 +83,17 @@ DocsGPT includes a suite of pre-built tools designed to expand its capabilities
},
{
title: 'Memory',
link: 'https://github.com/arc53/DocsGPT/blob/main/application/agents/tools/memory.py',
link: 'https://github.com/arc53/DocsGPT/blob/main/docsgpt/agents/tools/memory.py',
description: 'Stores and retrieves information across conversations through a per-user memory file directory.'
},
{
title: 'Notepad',
link: 'https://github.com/arc53/DocsGPT/blob/main/application/agents/tools/notes.py',
link: 'https://github.com/arc53/DocsGPT/blob/main/docsgpt/agents/tools/notes.py',
description: 'A single editable note. Supports viewing, overwriting, and string replacement.'
},
{
title: 'Todo List',
link: 'https://github.com/arc53/DocsGPT/blob/main/application/agents/tools/todo_list.py',
link: 'https://github.com/arc53/DocsGPT/blob/main/docsgpt/agents/tools/todo_list.py',
description: 'Manages todo items — creating, viewing, updating, and deleting todos.'
}
]}
+4 -4
View File
@@ -27,7 +27,7 @@ While DocsGPT offers a range of built-in tools and a versatile API Tool, there a
Before you begin, ensure you have:
* A solid understanding of Python programming.
* Familiarity with the DocsGPT project structure, particularly the `application/agents/tools/` directory where custom tools reside.
* Familiarity with the DocsGPT project structure, particularly the `docsgpt/agents/tools/` directory where custom tools reside.
* Basic knowledge of how APIs work, as many tools involve interacting with external or internal APIs.
* Your DocsGPT development environment set up. If not, please refer to the [Setting Up a Development Environment](/Deploying/Development-Environment) guide.
@@ -35,7 +35,7 @@ Before you begin, ensure you have:
Custom tools in DocsGPT are Python classes that inherit from a base `Tool` class and implement specific methods to define their behavior, capabilities, and configuration needs.
The **foundation** for all custom tools is the abstract base class, located in `application/agents/tools/base.py`. Your custom tool class **must** inherit from this class.
The **foundation** for all custom tools is the abstract base class, located in `docsgpt/agents/tools/base.py`. Your custom tool class **must** inherit from this class.
### Essential Methods to Implement
@@ -150,11 +150,11 @@ Your custom tool class needs to implement the following methods:
## Tool Registration and Discovery
DocsGPT's ToolManager (located in application/agents/tools/tool_manager.py) automatically discovers and loads tools.
DocsGPT's ToolManager (located in docsgpt/agents/tools/tool_manager.py) automatically discovers and loads tools.
As long as your custom tool:
1. Is placed in a Python file within the `application/agents/tools/` directory (and the filename is not `base.py` or starts with `__`).
1. Is placed in a Python file within the `docsgpt/agents/tools/` directory (and the filename is not `base.py` or starts with `__`).
2. Correctly inherits from the `Tool` base class.
3. Implements all the abstract methods (`execute_action`, `get_actions_metadata`, `get_config_requirements`).
+28 -6
View File
@@ -19,8 +19,8 @@ DocsGPT now runs embeddings through [FastEmbed](https://github.com/qdrant/fastem
**Your worker command does need one change.** Query embedding now runs on the Celery worker (`EMBEDDINGS_DELEGATE_TO_WORKER`, on by default), which keeps the API from loading a model of its own. If you start your worker with an explicit `-Q`, add the `embeddings` queue:
```diff
- celery -A application.app.celery worker -l INFO -Q docsgpt,parsing
+ celery -A application.app.celery worker -l INFO -Q docsgpt,parsing,embeddings
- celery -A docsgpt.app.celery worker -l INFO -Q docsgpt,parsing
+ celery -A docsgpt.app.celery worker -l INFO -Q docsgpt,parsing,embeddings
```
The bundled Compose and Kubernetes manifests already do this — pull them along with the code. Without it, every search blocks for `EMBEDDINGS_DELEGATE_TIMEOUT` (60s) and then answers with no retrieved context rather than raising, so the symptom is bad answers, not an error. To keep the model out of the worker too, set `EMBEDDINGS_BASE_URL`; to run the API on its own, set `EMBEDDINGS_DELEGATE_TO_WORKER=false`.
@@ -41,8 +41,8 @@ Set the model, then rebuild the vectors:
EMBEDDINGS_NAME=ibm-granite/granite-embedding-311m-multilingual-r2
# 2. Rebuild the vectors from the chunk text already in your index
docker compose exec backend python -m application.scripts.reembed --dry-run
docker compose exec backend python -m application.scripts.reembed
docker compose exec backend python -m docsgpt.scripts.reembed --dry-run
docker compose exec backend python -m docsgpt.scripts.reembed
```
Re-embedding reads the chunk text already stored in your index. It does not re-download, re-parse or re-chunk your documents, so no source files are needed and the run is proportional to index size, not corpus size. Both `pgvector` and `faiss` are supported.
@@ -90,10 +90,32 @@ need attention:
Staying on `all-mpnet-base-v2` is a supported choice — it remains in the model registry and in `setup.sh`. You only need this section if you want to move to granite.
</Callout>
## Backend package renamed to `docsgpt`
The backend's Python package is `docsgpt` (it was `application`), the name it
will carry on PyPI. For one release the old name keeps working through an
alias, so nothing breaks on upgrade, but update these before the alias goes:
- Entry points: `celery -A docsgpt.app.celery worker`,
`uvicorn docsgpt.asgi:asgi_app`, `python -m docsgpt.scripts.<name>`. The
`application.…` spellings still run and print a `FutureWarning`. The
compose files, Kubernetes manifests and setup scripts in the repository are
already updated; only custom copies need editing.
- Local image builds: the build context is the repository root, so use
`docker build -f docsgpt/Dockerfile .` (or the compose files, which do this).
- Celery task names changed with the package (`docsgpt.api.user.tasks.ingest`
and so on). A worker on this release also accepts the old names, so tasks
queued before the upgrade still run, and beat rewrites the periodic
schedule in Redis on start-up. The daily, weekly and monthly source-sync
timers restart from the upgrade, so the first sync after it can land later
than it would have (a monthly sync by up to a month). Nothing to do.
- Data directories do not move: the compose files keep your indexes, inputs
and vectors under `application/` in the checkout, where they already are.
## Check your version
```bash
docker compose exec backend python -c "from application.version import get_version; print(get_version())"
docker compose exec backend python -c "from docsgpt.version import get_version; print(get_version())"
```
Release notes: [changelog](/changelog). Tags: [GitHub releases](https://github.com/arc53/DocsGPT/releases).
@@ -135,7 +157,7 @@ Full manifests: [Kubernetes deployment guide](/Deploying/Kubernetes-Deploying).
Alembic migrations run on worker startup. To apply manually:
```bash
docker compose exec backend alembic -c application/alembic.ini upgrade head
docker compose exec backend alembic -c docsgpt/alembic.ini upgrade head
```
`upgrade head` is idempotent.
+5 -5
View File
@@ -126,7 +126,7 @@ reconnect HTTP errored. Common cases:
- The user's JWT rotated mid-stream → 401 on the GET. Frontend
doesn't auto-refresh; the user reloads.
- The user is on a different host than the API and CORS is rejecting
the GET → check `application/asgi.py` allow-headers.
the GET → check `docsgpt/asgi.py` allow-headers.
### D. "The dev install never delivers any notifications at all"
@@ -316,7 +316,7 @@ redis-cli -n 2 DEL user:<id>:stream
## Settings reference
Everything in `application/core/settings.py`:
Everything in `docsgpt/core/settings.py`:
| Setting | Default | Purpose |
| --------------------------------------------- | ------- | --------------------------------------------- |
@@ -364,16 +364,16 @@ re-surface UX is too aggressive for v2.
The chat-stream reconnect reader `GET /api/messages/<id>/events`
is a native-async Starlette route mounted in
`application/asgi.py`, not a Flask route. Plain `flask run`
`docsgpt/asgi.py`, not a Flask route. Plain `flask run`
serves only the WSGI Flask app, so under it that endpoint 404s
and reconnect-after-disconnect can't resume. Run the backend via
`uvicorn application.asgi:asgi_app --reload` (or the production
`uvicorn docsgpt.asgi:asgi_app --reload` (or the production
gunicorn uvicorn-worker) to exercise it.
### Werkzeug doesn't auto-reload route files
The dev server (`flask run`) doesn't watch
`application/api/events/routes.py` for changes by default.
`docsgpt/api/events/routes.py` for changes by default.
After editing the route, restart Flask manually — `--reload`
isn't on. (Production gunicorn reloads via deploy.)
+175
View File
@@ -0,0 +1,175 @@
# DocsGPT backend image.
#
# Build args:
# EXTRAS comma-separated optional extras to bake in, matching the
# pyproject extras / requirements-<extra>.txt files:
# docling (layout-model parser + OCR backend), milvus.
# INSTALL_DOCLING legacy alias for EXTRAS=docling (setup.sh writes it).
# INSTALL_TESSERACT bake the tesseract binary for OCR_ENGINE=tesseract.
# EMBEDDINGS_PREFETCH registry names of the embedding models to bake; empty
# bakes both defaults (mpnet for upgrades, granite for
# new installs).
#
# Everything the default configuration needs is inside the image: embedding
# models, their tokenizers, tiktoken's encoding and, with the docling extra,
# docling's layout/table/OCR models. `python -m docsgpt.scripts.verify_offline`
# under `docker run --network none` proves it.
FROM ubuntu:24.04 AS builder
ENV DEBIAN_FRONTEND=noninteractive
# Ubuntu 24.04 ships Python 3.12 in its main archive: no PPA needed. Every pin
# resolves to a wheel, so no compiler toolchain either.
RUN apt-get update && \
apt-get install -y --no-install-recommends python3.12 python3.12-venv ca-certificates && \
rm -rf /var/lib/apt/lists/*
# Build context is the repository root (see .dockerignore there):
# docker build -f docsgpt/Dockerfile .
COPY docsgpt/requirements.txt docsgpt/requirements-docling.txt docsgpt/requirements-milvus.txt ./
RUN python3.12 -m venv /venv
ENV PATH="/venv/bin:$PATH"
RUN pip install --no-cache-dir --upgrade pip && \
pip install --no-cache-dir --only-binary=:all: -r requirements.txt
# Optional extras. Each requirements-<extra>.txt is exported from the same
# lock as requirements.txt, so installing it on top only adds the extra's
# packages. The docling file takes torch from the CPU-only PyTorch index.
# Not wheels-only: docling's antlr4 runtime ships as a pure-Python sdist.
ARG EXTRAS=""
ARG INSTALL_DOCLING=false
RUN set -e; \
extras="$EXTRAS"; \
if [ "$INSTALL_DOCLING" = "true" ]; then extras="$extras,docling"; fi; \
for extra in $(echo "$extras" | tr ',' ' '); do \
echo "Installing extra: $extra"; \
pip install --no-cache-dir -r "requirements-$extra.txt"; \
done
# google-api-python-client bundles discovery documents for ~600 Google APIs
# (99 MB). The application builds one client, Drive v3; keep only its document.
# Building another API's client needs its file back, or static_discovery=False.
RUN find /venv/lib/python3.12/site-packages/googleapiclient/discovery_cache/documents \
-type f ! -name 'drive.v3.json' -delete
FROM ubuntu:24.04 AS final
ENV DEBIAN_FRONTEND=noninteractive
RUN apt-get update && \
apt-get install -y --no-install-recommends python3.12 poppler-utils ca-certificates && \
ln -s /usr/bin/python3.12 /usr/bin/python && \
rm -rf /var/lib/apt/lists/*
# opencv (rapidocr, part of the docling extra) needs libGL at import time.
ARG EXTRAS=""
ARG INSTALL_DOCLING=false
RUN if [ "$INSTALL_DOCLING" = "true" ] || echo ",$EXTRAS," | grep -q ",docling,"; then \
apt-get update && \
apt-get install -y --no-install-recommends libgl1 libglib2.0-0 && \
rm -rf /var/lib/apt/lists/*; \
fi
# Optional tesseract OCR engine (OCR_ENABLED=true with OCR_ENGINE=tesseract,
# the default engine); ~35 MB of system packages. Extra language packs are a
# deployment concern (apt: tesseract-ocr-<lang>, then list them in OCR_LANGS).
# A DeepSeek-OCR endpoint (OCR_ENGINE=deepseek) needs none of this.
ARG INSTALL_TESSERACT=false
RUN if [ "$INSTALL_TESSERACT" = "true" ]; then \
apt-get update && \
apt-get install -y --no-install-recommends tesseract-ocr tesseract-ocr-eng && \
rm -rf /var/lib/apt/lists/*; \
fi
LABEL org.opencontainers.image.source="https://github.com/arc53/DocsGPT" \
org.opencontainers.image.title="DocsGPT" \
org.opencontainers.image.description="DocsGPT backend: API and Celery worker" \
org.opencontainers.image.licenses="MIT"
WORKDIR /app
# The process user owns /app so the model prefetch below can run as it: an
# unprivileged prefetch writes the model files with the right owner up front,
# instead of a trailing chown -R that rewrites every model file into a second
# layer.
RUN groupadd -r appuser && \
useradd -r -g appuser -d /app -s /sbin/nologin -c "Docker image user" appuser && \
chown appuser:appuser /app && \
install -d -o appuser -g appuser /app/models /app/docsgpt
COPY --from=builder /venv /venv
# Every cache the application reads at run time lives under /app/models and is
# filled at build time:
# EMBEDDINGS_CACHE_DIR / HF_HUB_CACHE FastEmbed models and their tokenizers
# (chunking reads tokenizer.json from
# the same hub-layout snapshot)
# TIKTOKEN_CACHE_DIR cl100k_base for token accounting
# DOCLING_ARTIFACTS_PATH docling's models (docling extra only)
ENV EMBEDDINGS_CACHE_DIR=/app/models \
HF_HUB_CACHE=/app/models \
TIKTOKEN_CACHE_DIR=/app/models/tiktoken \
DOCLING_ARTIFACTS_PATH=/app/models/docling \
HF_HUB_DISABLE_TELEMETRY=1 \
PATH="/venv/bin:$PATH"
# Only the modules the prefetch imports are copied first, so an unrelated
# source edit does not invalidate the model layer.
COPY --chown=appuser:appuser docsgpt/__init__.py /app/docsgpt/__init__.py
COPY --chown=appuser:appuser docsgpt/scripts/__init__.py docsgpt/scripts/prefetch_models.py /app/docsgpt/scripts/
COPY --chown=appuser:appuser docsgpt/vectorstore/__init__.py docsgpt/vectorstore/model_registry.py /app/docsgpt/vectorstore/
USER appuser
ARG EMBEDDINGS_PREFETCH=""
RUN PYTHONPATH=/app python -m docsgpt.scripts.prefetch_models ${EMBEDDINGS_PREFETCH} && \
rm -rf /app/models/.locks /app/.cache
# docling downloads its layout, table-structure and OCR models on first parse;
# bake them so the docling variant is as self-contained as the default image.
RUN if python -c "import docling" 2>/dev/null; then \
docling-tools models download --output-dir /app/models/docling layout tableformer rapidocr && \
rm -rf /app/.cache; \
fi
COPY --chown=appuser:appuser docsgpt /app/docsgpt
# One-release alias so `-A application.app.celery` style entry points keep working.
COPY --chown=appuser:appuser application/__init__.py /app/application/__init__.py
# Runtime data directories, owned by the process user so a named volume
# mounted on them (docker-compose-standalone.yaml) inherits that ownership
# and uploads work without running the container as root.
RUN mkdir -p /app/docsgpt/inputs/local /app/inputs /app/indexes /app/vectors
ENV FLASK_APP=app.py
# Thread caps. onnxruntime (FastEmbed) ignores OMP_NUM_THREADS and sizes its
# pool to the host's core count, which a CPU-limited container still reports;
# EMBEDDINGS_THREADS pins it the way OMP_NUM_THREADS pinned torch before.
ENV MALLOC_ARENA_MAX=2 \
OMP_NUM_THREADS=4 \
MKL_NUM_THREADS=4 \
OPENBLAS_NUM_THREADS=4 \
EMBEDDINGS_THREADS=4
EXPOSE 7091
# BoundedDrainUvicornWorker makes max_requests recycles safe with held-open SSE
# connections (see docsgpt/gunicorn_worker.py); with recycles now safe,
# --max-requests is raised (kept for memory hygiene) to cut churn.
CMD ["gunicorn", \
"-w", "1", \
"-k", "docsgpt.gunicorn_worker.BoundedDrainUvicornWorker", \
"--bind", "0.0.0.0:7091", \
"--timeout", "180", \
"--graceful-timeout", "120", \
"--keep-alive", "5", \
"--worker-tmp-dir", "/dev/shm", \
"--max-requests", "5000", \
"--max-requests-jitter", "500", \
"--config", "docsgpt/gunicorn_conf.py", \
"docsgpt.asgi:asgi_app"]
File renamed without changes.
File renamed without changes.
@@ -1,9 +1,9 @@
import logging
from application.agents.agentic_agent import AgenticAgent
from application.agents.classic_agent import ClassicAgent
from application.agents.research_agent import ResearchAgent
from application.agents.workflow_agent import WorkflowAgent
from docsgpt.agents.agentic_agent import AgenticAgent
from docsgpt.agents.classic_agent import ClassicAgent
from docsgpt.agents.research_agent import ResearchAgent
from docsgpt.agents.workflow_agent import WorkflowAgent
logger = logging.getLogger(__name__)
@@ -1,10 +1,10 @@
import logging
from typing import Dict, Generator, Optional
from application.agents.base import BaseAgent
from application.agents.tools.internal_search import add_internal_search_tool
from application.agents.tools.wiki import add_wiki_tool
from application.logging import LogContext
from docsgpt.agents.base import BaseAgent
from docsgpt.agents.tools.internal_search import add_internal_search_tool
from docsgpt.agents.tools.wiki import add_wiki_tool
from docsgpt.logging import LogContext
logger = logging.getLogger(__name__)
@@ -6,30 +6,30 @@ from abc import ABC, abstractmethod
from datetime import datetime, timezone
from typing import Any, Dict, Generator, List, Optional
from application.agents.tool_executor import (
from docsgpt.agents.tool_executor import (
ToolExecutor,
result_status,
truncate_tool_result,
)
from application.core.json_schema_utils import (
from docsgpt.core.json_schema_utils import (
JsonSchemaValidationError,
normalize_json_schema_payload,
)
from application.core.settings import settings
from application.llm.handlers.base import (
from docsgpt.core.settings import settings
from docsgpt.llm.handlers.base import (
ToolCall,
_bound_tool_response_for_llm,
)
from application.guardrails.config import DEFAULT_BLOCK_MESSAGE as GUARDRAIL_DEFAULT_MESSAGE
from application.guardrails.runtime import (
from docsgpt.guardrails.config import DEFAULT_BLOCK_MESSAGE as GUARDRAIL_DEFAULT_MESSAGE
from docsgpt.guardrails.runtime import (
build_engine as build_guardrail_engine,
resolve_config as resolve_guardrails_config,
)
from application.guardrails.stream import StreamingOutputGuard
from application.guardrails.types import Action, Stage, resolve_tool_result
from application.llm.handlers.handler_creator import LLMHandlerCreator
from application.llm.llm_creator import LLMCreator
from application.logging import build_stack_data, log_activity, LogContext
from docsgpt.guardrails.stream import StreamingOutputGuard
from docsgpt.guardrails.types import Action, Stage, resolve_tool_result
from docsgpt.llm.handlers.handler_creator import LLMHandlerCreator
from docsgpt.llm.llm_creator import LLMCreator
from docsgpt.logging import build_stack_data, log_activity, LogContext
logger = logging.getLogger(__name__)
@@ -424,7 +424,7 @@ class BaseAgent(ABC):
return meta["response_id"]
budget = getattr(settings, "OPENAI_RESPONSES_CHAIN_BUDGET_TOKENS", None)
if not budget:
from application.core.model_utils import get_token_limit
from docsgpt.core.model_utils import get_token_limit
budget = get_token_limit(
getattr(self, "model_id", None),
@@ -677,13 +677,13 @@ class BaseAgent(ABC):
# ---- Context / token management ----
def _calculate_current_context_tokens(self, messages: List[Dict]) -> int:
from application.api.answer.services.compression.token_counter import (
from docsgpt.api.answer.services.compression.token_counter import (
TokenCounter,
)
return TokenCounter.count_message_tokens(messages)
def _check_context_limit(self, messages: List[Dict]) -> bool:
from application.core.model_utils import get_token_limit
from docsgpt.core.model_utils import get_token_limit
try:
current_tokens = self._calculate_current_context_tokens(messages)
@@ -705,7 +705,7 @@ class BaseAgent(ABC):
return False
def _validate_context_size(self, messages: List[Dict]) -> None:
from application.core.model_utils import get_token_limit
from docsgpt.core.model_utils import get_token_limit
current_tokens = self._calculate_current_context_tokens(messages)
self.current_token_count = current_tokens
@@ -728,7 +728,7 @@ class BaseAgent(ABC):
)
def _truncate_text_middle(self, text: str, max_tokens: int) -> str:
from application.utils import num_tokens_from_string
from docsgpt.utils import num_tokens_from_string
current_tokens = num_tokens_from_string(text)
if current_tokens <= max_tokens:
@@ -767,8 +767,8 @@ class BaseAgent(ABC):
(the usual culprit) and raises when even that cannot fit — BEFORE
the usage decorators run, so a hopeless payload costs nothing.
"""
from application.core.model_utils import get_token_limit
from application.utils import num_tokens_from_string
from docsgpt.core.model_utils import get_token_limit
from docsgpt.utils import num_tokens_from_string
context_limit = get_token_limit(
self.model_id, user_id=self.model_user_id or self.user
@@ -847,7 +847,7 @@ class BaseAgent(ABC):
"""
if getattr(self, "prompt_embeds_documents", False):
return ""
from application.api.answer.services.prompt_renderer import (
from docsgpt.api.answer.services.prompt_renderer import (
format_docs_for_prompt,
)
@@ -887,7 +887,7 @@ class BaseAgent(ABC):
attacker-influenceable material. The prompt was rendered before the
agent ran, so the verdict is applied by patching it here.
"""
from application.api.answer.services.prompt_renderer import (
from docsgpt.api.answer.services.prompt_renderer import (
format_docs_for_prompt,
)
@@ -959,7 +959,7 @@ class BaseAgent(ABC):
"""Merge the cached InternalSearchTool's docs into ``retrieved_docs``,
deduped, preserving any pre-fetched docs so a mixed-exposure agent cites
both pre-fetched and tool-retrieved sources (not just the tool's)."""
from application.agents.tools.internal_search import INTERNAL_TOOL_ID
from docsgpt.agents.tools.internal_search import INTERNAL_TOOL_ID
executor = getattr(self, "tool_executor", None)
loaded = getattr(executor, "_loaded_tools", None) or {}
@@ -1004,8 +1004,8 @@ class BaseAgent(ABC):
query: str,
) -> List[Dict]:
"""Build messages using pre-rendered system prompt"""
from application.core.model_utils import get_token_limit
from application.utils import num_tokens_from_string
from docsgpt.core.model_utils import get_token_limit
from docsgpt.utils import num_tokens_from_string
# Retrieval controls run inside _build_document_block for the usual
# path; a prompt that embeds the documents skips that block entirely,
@@ -1197,7 +1197,7 @@ class BaseAgent(ABC):
history: List[Dict],
max_tokens: int,
) -> List[Dict]:
from application.utils import num_tokens_from_string
from docsgpt.utils import num_tokens_from_string
if not history or max_tokens <= 0:
return []
@@ -1,10 +1,10 @@
import logging
from typing import Dict, Generator, Optional
from application.agents.base import BaseAgent
from application.agents.tools.internal_search import add_internal_search_tool
from application.agents.tools.wiki import add_wiki_tool
from application.logging import LogContext
from docsgpt.agents.base import BaseAgent
from docsgpt.agents.tools.internal_search import add_internal_search_tool
from docsgpt.agents.tools.wiki import add_wiki_tool
from docsgpt.logging import LogContext
logger = logging.getLogger(__name__)
@@ -8,7 +8,7 @@ import logging
import uuid
from typing import Any, Dict, List, Optional
from application.core.settings import settings
from docsgpt.core.settings import settings
logger = logging.getLogger(__name__)
@@ -61,15 +61,15 @@ _builtin_loaded_cache: Dict[tuple, List[str]] = {}
def _load_tool(tool_name: str) -> Optional[Any]:
"""Return a metadata-only instance of a tool, or None if it has no class."""
# Imports just the named module (not the whole package) — avoids the
# circular import via ``mcp_tool`` → ``application.api.user``.
# circular import via ``mcp_tool`` → ``docsgpt.api.user``.
if tool_name in _tool_cache:
return _tool_cache[tool_name]
from application.agents.tools.base import Tool
from docsgpt.agents.tools.base import Tool
instance: Optional[Any] = None
try:
module = importlib.import_module(f"application.agents.tools.{tool_name}")
module = importlib.import_module(f"docsgpt.agents.tools.{tool_name}")
except ModuleNotFoundError:
_tool_cache[tool_name] = None
return None
@@ -5,19 +5,19 @@ from __future__ import annotations
import logging
from typing import Any, Dict, Iterable, List, Optional
from application.agents.agent_creator import AgentCreator
from application.agents.tool_executor import ToolExecutor
from application.api.answer.services.prompt_renderer import (
from docsgpt.agents.agent_creator import AgentCreator
from docsgpt.agents.tool_executor import ToolExecutor
from docsgpt.api.answer.services.prompt_renderer import (
PromptRenderer,
format_docs_for_prompt,
prompt_embeds_documents,
resolve_prompt_skeleton,
)
from application.api.answer.services.stream_processor import get_prompt
from application.core.settings import settings
from application.retriever.retriever_creator import RetrieverCreator
from application.storage.db.repositories.sources import SourcesRepository
from application.storage.db.session import db_readonly
from docsgpt.api.answer.services.stream_processor import get_prompt
from docsgpt.core.settings import settings
from docsgpt.retriever.retriever_creator import RetrieverCreator
from docsgpt.storage.db.repositories.sources import SourcesRepository
from docsgpt.storage.db.session import db_readonly
logger = logging.getLogger(__name__)
@@ -70,13 +70,13 @@ def run_agent_headless(
conversation_id: Optional[str] = None,
) -> Dict[str, Any]:
"""Run an agent with no live client; returns a structured outcome dict."""
from application.core.model_utils import (
from docsgpt.core.model_utils import (
get_api_key_for_provider,
get_default_model_id,
get_provider_from_model_id,
validate_model_id,
)
from application.utils import calculate_doc_token_budget
from docsgpt.utils import calculate_doc_token_budget
owner = _resolve_owner(agent_config)
if not owner:
@@ -210,7 +210,7 @@ def run_agent_headless(
# by raising. Dropping it here (as this loop used to) makes a broken run
# indistinguishable from one that simply had nothing to say, and the
# caller records it as a success. Mirrors the sentinel in
# ``application/logging.py`` so an error carrying no message is still
# ``docsgpt/logging.py`` so an error carrying no message is still
# truthy instead of reading as "ok".
if event.get("type") == "error":
stream_error = str(event.get("error") or "")[:500] or "unspecified"
@@ -4,15 +4,15 @@ import os
import time
from typing import Dict, Generator, List, Optional
from application.agents.base import BaseAgent
from application.agents.tool_executor import ToolExecutor
from application.agents.tools.internal_search import (
from docsgpt.agents.base import BaseAgent
from docsgpt.agents.tool_executor import ToolExecutor
from docsgpt.agents.tools.internal_search import (
INTERNAL_TOOL_ID,
add_internal_search_tool,
)
from application.agents.tools.wiki import add_wiki_tool
from application.agents.tools.think import THINK_TOOL_ENTRY, THINK_TOOL_ID
from application.logging import LogContext
from docsgpt.agents.tools.wiki import add_wiki_tool
from docsgpt.agents.tools.think import THINK_TOOL_ENTRY, THINK_TOOL_ID
from docsgpt.logging import LogContext
logger = logging.getLogger(__name__)
@@ -673,7 +673,7 @@ class ResearchAgent(BaseAgent):
)
if log_context:
from application.logging import build_stack_data
from docsgpt.logging import build_stack_data
log_context.stacks.append(
{"component": "synthesis_llm", "data": build_stack_data(self.llm)}
File renamed without changes.
@@ -6,25 +6,25 @@ from typing import Any, Dict, List, Optional, Tuple
from sqlalchemy.exc import IntegrityError
from application.agents.default_tools import (
from docsgpt.agents.default_tools import (
BUILTIN_AGENT_TOOLS,
is_headless_excluded_tool,
is_synthesized_tool_id,
resolve_tool_by_id,
synthesized_default_tools,
)
from application.agents.tools.tool_action_parser import ToolActionParser
from application.agents.tools.tool_manager import ToolManager
from application.guardrails.types import Stage as GuardrailStage, resolve_tool_result
from application.security.encryption import decrypt_credentials
from application.storage.db.base_repository import looks_like_uuid
from application.storage.db.repositories.agents import AgentsRepository
from application.storage.db.repositories.tool_call_attempts import (
from docsgpt.agents.tools.tool_action_parser import ToolActionParser
from docsgpt.agents.tools.tool_manager import ToolManager
from docsgpt.guardrails.types import Stage as GuardrailStage, resolve_tool_result
from docsgpt.security.encryption import decrypt_credentials
from docsgpt.storage.db.base_repository import looks_like_uuid
from docsgpt.storage.db.repositories.agents import AgentsRepository
from docsgpt.storage.db.repositories.tool_call_attempts import (
ToolCallAttemptsRepository,
)
from application.storage.db.repositories.user_tools import UserToolsRepository
from application.storage.db.repositories.users import UsersRepository
from application.storage.db.session import db_readonly, db_session
from docsgpt.storage.db.repositories.user_tools import UserToolsRepository
from docsgpt.storage.db.repositories.users import UsersRepository
from docsgpt.storage.db.session import db_readonly, db_session
logger = logging.getLogger(__name__)
@@ -51,7 +51,7 @@ def _dedupable_tool_names() -> frozenset:
Only these are safe to collapse — an MCP or user-added tool may
legitimately appear more than once under a single name.
"""
from application.core.settings import settings
from docsgpt.core.settings import settings
return frozenset(BUILTIN_AGENT_TOOLS) | frozenset(getattr(settings, "DEFAULT_CHAT_TOOLS", None) or [])
@@ -846,7 +846,7 @@ class ToolExecutor:
silently bypasses the prompt — not even via the headless allowlist.
"""
try:
from application.agents.tools.remote_device import RemoteDeviceTool
from docsgpt.agents.tools.remote_device import RemoteDeviceTool
tool = RemoteDeviceTool(
config=tool_data.get("config") or {},
@@ -872,7 +872,7 @@ class ToolExecutor:
error so a misconfigured tool never silently runs untrusted code.
"""
try:
from application.agents.tools.code_executor import CodeExecutorTool
from docsgpt.agents.tools.code_executor import CodeExecutorTool
tool = CodeExecutorTool(
tool_config=tool_data.get("config") or {},
@@ -6,12 +6,12 @@ from urllib.parse import quote, urlencode
import requests
from application.agents.tools.api_body_serializer import (
from docsgpt.agents.tools.api_body_serializer import (
ContentType,
RequestBodySerializer,
)
from application.agents.tools.base import Tool
from application.security.safe_url import UnsafeUserUrlError, pinned_request
from docsgpt.agents.tools.base import Tool
from docsgpt.security.safe_url import UnsafeUserUrlError, pinned_request
logger = logging.getLogger(__name__)
@@ -17,17 +17,17 @@ import logging
import uuid
from typing import Any, Dict, List, Optional, Tuple
from application.agents.tools.artifact_ref import resolve_artifact_id
from application.agents.tools.base import Tool
from application.core.settings import settings
from application.sandbox.artifacts_capture import (
from docsgpt.agents.tools.artifact_ref import resolve_artifact_id
from docsgpt.agents.tools.base import Tool
from docsgpt.core.settings import settings
from docsgpt.sandbox.artifacts_capture import (
QuotaExceeded,
append_artifact_version,
persist_new_artifact,
)
from application.sandbox.sandbox_creator import SandboxCreator
from application.storage.db.repositories.artifacts import ArtifactsRepository
from application.storage.db.session import db_readonly
from docsgpt.sandbox.sandbox_creator import SandboxCreator
from docsgpt.storage.db.repositories.artifacts import ArtifactsRepository
from docsgpt.storage.db.session import db_readonly
logger = logging.getLogger(__name__)
@@ -13,7 +13,7 @@ from __future__ import annotations
import re
from typing import Any, Optional
from application.storage.db.base_repository import looks_like_uuid
from docsgpt.storage.db.base_repository import looks_like_uuid
_REF_RE = re.compile(r"^[Aa](\d+)$")
@@ -12,12 +12,12 @@ from __future__ import annotations
import logging
from typing import Any, Dict, List, Optional
from application.core.settings import settings
from application.sandbox.artifacts_capture import QuotaExceeded, persist_new_artifact
from application.storage.db.repositories.artifacts import ArtifactsRepository
from application.storage.db.repositories.attachments import AttachmentsRepository
from application.storage.db.session import db_readonly
from application.storage.storage_creator import StorageCreator
from docsgpt.core.settings import settings
from docsgpt.sandbox.artifacts_capture import QuotaExceeded, persist_new_artifact
from docsgpt.storage.db.repositories.artifacts import ArtifactsRepository
from docsgpt.storage.db.repositories.attachments import AttachmentsRepository
from docsgpt.storage.db.session import db_readonly
from docsgpt.storage.storage_creator import StorageCreator
logger = logging.getLogger(__name__)
File renamed without changes.
@@ -2,7 +2,7 @@ import logging
import requests
from application.agents.tools.base import Tool
from docsgpt.agents.tools.base import Tool
logger = logging.getLogger(__name__)
@@ -6,32 +6,32 @@ import logging
import re
from typing import Any, Dict, List, Optional, Tuple
from application.agents.tools.artifact_ref import resolve_artifact_id
from application.agents.tools.attachment_bridge import (
from docsgpt.agents.tools.artifact_ref import resolve_artifact_id
from docsgpt.agents.tools.attachment_bridge import (
AttachmentBridgeError,
bridge_attachment,
match_attachment,
)
from application.agents.tools.base import Tool
from application.core.settings import settings
from application.sandbox.artifacts_capture import (
from docsgpt.agents.tools.base import Tool
from docsgpt.core.settings import settings
from docsgpt.sandbox.artifacts_capture import (
MAX_CAPTURED_FILES,
capture_artifacts,
snapshot_signatures,
unique_input_path,
)
from application.sandbox.artifacts_capture import (
from docsgpt.sandbox.artifacts_capture import (
infer_mime as _infer_mime,
)
from application.sandbox.artifacts_capture import (
from docsgpt.sandbox.artifacts_capture import (
kind_for_mime as _kind_for_mime,
)
from application.sandbox.base import ExecResult
from application.sandbox.sandbox_creator import SandboxCreator
from application.storage.db.repositories.artifacts import ArtifactsRepository
from application.storage.db.session import db_readonly
from application.storage.storage_creator import StorageCreator
from application.utils import safe_filename
from docsgpt.sandbox.base import ExecResult
from docsgpt.sandbox.sandbox_creator import SandboxCreator
from docsgpt.storage.db.repositories.artifacts import ArtifactsRepository
from docsgpt.storage.db.session import db_readonly
from docsgpt.storage.storage_creator import StorageCreator
from docsgpt.utils import safe_filename
logger = logging.getLogger(__name__)
@@ -1,5 +1,5 @@
import requests
from application.agents.tools.base import Tool
from docsgpt.agents.tools.base import Tool
class CryptoPriceTool(Tool):
@@ -2,7 +2,7 @@ import logging
import time
from typing import Any, Dict, Optional
from application.agents.tools.base import Tool
from docsgpt.agents.tools.base import Tool
logger = logging.getLogger(__name__)
@@ -2,10 +2,10 @@ import json
import logging
from typing import Dict, List, Optional
from application.agents.tools.base import Tool
from application.core.settings import settings
from application.retriever.dispatcher import build_dispatcher
from application.retriever.retriever_creator import RetrieverCreator
from docsgpt.agents.tools.base import Tool
from docsgpt.core.settings import settings
from docsgpt.retriever.dispatcher import build_dispatcher
from docsgpt.retriever.retriever_creator import RetrieverCreator
logger = logging.getLogger(__name__)
@@ -78,10 +78,10 @@ class InternalSearchTool(Tool):
# Per-operation session: this tool runs inside the answer
# generator hot path, so we open a short-lived read
# connection for the batch lookup and release immediately.
from application.storage.db.repositories.sources import (
from docsgpt.storage.db.repositories.sources import (
SourcesRepository,
)
from application.storage.db.session import db_readonly
from docsgpt.storage.db.session import db_readonly
if isinstance(active_docs, str):
active_docs = [active_docs]
@@ -402,7 +402,7 @@ def sources_have_directory_structure(source: Dict) -> bool:
# sites are updated to propagate user context.
from sqlalchemy import text as _text
from application.storage.db.session import db_readonly
from docsgpt.storage.db.session import db_readonly
if isinstance(active_docs, str):
active_docs = [active_docs]
@@ -19,13 +19,13 @@ from mcp.shared.auth import OAuthClientInformationFull, OAuthClientMetadata, OAu
from pydantic import AnyHttpUrl, ValidationError
from redis import Redis
from application.agents.tools.base import Tool
from application.api.user.tasks import mcp_oauth_task
from application.cache import get_redis_instance
from application.core.settings import settings
from application.core.url_validation import SSRFError, validate_url
from application.events.keys import stream_key
from application.security.encryption import decrypt_credentials
from docsgpt.agents.tools.base import Tool
from docsgpt.api.user.tasks import mcp_oauth_task
from docsgpt.cache import get_redis_instance
from docsgpt.core.settings import settings
from docsgpt.core.url_validation import SSRFError, validate_url
from docsgpt.events.keys import stream_key
from docsgpt.security.encryption import decrypt_credentials
logger = logging.getLogger(__name__)
@@ -856,10 +856,10 @@ class DBTokenStorage(TokenStorage):
def _fetch_session_data(self) -> dict:
"""Read the JSONB ``session_data`` blob for this MCP server row."""
from application.storage.db.repositories.connector_sessions import (
from docsgpt.storage.db.repositories.connector_sessions import (
ConnectorSessionsRepository,
)
from application.storage.db.session import db_readonly
from docsgpt.storage.db.session import db_readonly
base_url = self.get_base_url(self.server_url)
with db_readonly() as conn:
@@ -893,10 +893,10 @@ class DBTokenStorage(TokenStorage):
the scalar column — ``get_by_user_and_server_url`` needs that to
resolve the row (``NULL = 'https://...'`` is UNKNOWN in SQL).
"""
from application.storage.db.repositories.connector_sessions import (
from docsgpt.storage.db.repositories.connector_sessions import (
ConnectorSessionsRepository,
)
from application.storage.db.session import db_session
from docsgpt.storage.db.session import db_session
base_url = self.get_base_url(self.server_url)
with db_session() as conn:
@@ -905,10 +905,10 @@ class DBTokenStorage(TokenStorage):
)
def _delete(self) -> None:
from application.storage.db.repositories.connector_sessions import (
from docsgpt.storage.db.repositories.connector_sessions import (
ConnectorSessionsRepository,
)
from application.storage.db.session import db_session
from docsgpt.storage.db.session import db_session
with db_session() as conn:
ConnectorSessionsRepository(conn).delete(
@@ -980,7 +980,7 @@ class DBTokenStorage(TokenStorage):
"""
from sqlalchemy import text
from application.storage.db.session import db_session
from docsgpt.storage.db.session import db_session
def _delete_all() -> None:
with db_session() as conn:
@@ -4,8 +4,8 @@ import uuid
from .base import Tool
from .path_utils import validate_tool_path
from application.storage.db.repositories.memories import MemoriesRepository
from application.storage.db.session import db_readonly, db_session
from docsgpt.storage.db.repositories.memories import MemoriesRepository
from docsgpt.storage.db.session import db_readonly, db_session
logger = logging.getLogger(__name__)
@@ -59,7 +59,7 @@ class MemoryTool(Tool):
tool_id,
)
return False
from application.storage.db.base_repository import looks_like_uuid
from docsgpt.storage.db.base_repository import looks_like_uuid
if not looks_like_uuid(tool_id):
logger.debug(
@@ -2,8 +2,8 @@ from typing import Any, Dict, List, Optional
import uuid
from .base import Tool
from application.storage.db.repositories.notes import NotesRepository
from application.storage.db.session import db_readonly, db_session
from docsgpt.storage.db.repositories.notes import NotesRepository
from docsgpt.storage.db.session import db_readonly, db_session
# Stable synthetic title used in the Postgres ``notes.title`` column.
@@ -55,7 +55,7 @@ class NotesTool(Tool):
return False
if tool_id.startswith("default_"):
return False
from application.storage.db.base_repository import looks_like_uuid
from docsgpt.storage.db.base_repository import looks_like_uuid
return looks_like_uuid(tool_id)
@@ -1,5 +1,5 @@
from application.agents.tools.base import Tool
from application.security.safe_url import UnsafeUserUrlError, pinned_request
from docsgpt.agents.tools.base import Tool
from docsgpt.security.safe_url import UnsafeUserUrlError, pinned_request
class NtfyTool(Tool):
"""
File renamed without changes.
@@ -2,7 +2,7 @@ import logging
import psycopg
from application.agents.tools.base import Tool
from docsgpt.agents.tools.base import Tool
logger = logging.getLogger(__name__)
@@ -19,20 +19,20 @@ from typing import Any, Callable, Dict, List, Optional
from celery import current_task
from application.agents.tools.artifact_ref import resolve_artifact_id
from application.agents.tools.attachment_bridge import (
from docsgpt.agents.tools.artifact_ref import resolve_artifact_id
from docsgpt.agents.tools.attachment_bridge import (
AttachmentBridgeError,
bridge_attachment,
match_attachment,
)
from application.agents.tools.base import Tool
from application.core.json_schema_utils import (
from docsgpt.agents.tools.base import Tool
from docsgpt.core.json_schema_utils import (
JsonSchemaValidationError,
normalize_json_schema_payload,
)
from application.core.settings import settings
from application.storage.db.repositories.artifacts import ArtifactsRepository
from application.storage.db.session import db_readonly
from docsgpt.core.settings import settings
from docsgpt.storage.db.repositories.artifacts import ArtifactsRepository
from docsgpt.storage.db.session import db_readonly
logger = logging.getLogger(__name__)
@@ -223,7 +223,7 @@ class ReadDocumentTool(Tool):
until the OUTER task's limit (webhook runs have none) kills the whole agent run.
"""
parent = self._parent()
from application.api.user.tasks import parse_timeout_for_size
from docsgpt.api.user.tasks import parse_timeout_for_size
# OCR cost scales with pages, so the parse window grows with the document's size
# (floored at DOCUMENT_PARSE_TIMEOUT).
@@ -232,7 +232,7 @@ class ReadDocumentTool(Tool):
# ``current_task`` is a Celery proxy: truthy only while this runs inside a worker task,
# falsy in the web process (the bare proxy is NOT identity-None, so test truthiness).
if current_task:
from application.worker import run_parse_document
from docsgpt.worker import run_parse_document
try:
result = self._run_inline_bounded(
@@ -250,7 +250,7 @@ class ReadDocumentTool(Tool):
from celery.exceptions import TimeoutError as CeleryTimeoutError
from application.api.user.tasks import parse_document, parse_task_time_limits
from docsgpt.api.user.tasks import parse_document, parse_task_time_limits
# The task's per-call time limits are raised to match the awaited window: bound to
# the base timeout at import, the worker would otherwise self-terminate a large
@@ -350,7 +350,7 @@ class ReadDocumentTool(Tool):
# are non-daemon and registered with ``concurrent.futures``' atexit hook,
# which joins them -- so an abandoned parse would hold up worker
# shutdown for the rest of its (size-scaled) window. Same reasoning as
# application/guardrails/engine.py. ``shutdown(cancel_futures=True)``
# docsgpt/guardrails/engine.py. ``shutdown(cancel_futures=True)``
# is not an alternative: it only drops queued work items, never the one
# already running.
slot: Dict[str, Any] = {}
@@ -2,8 +2,8 @@ import codecs
from markdownify import markdownify
from application.agents.tools.base import Tool
from application.security.safe_url import (
from docsgpt.agents.tools.base import Tool
from docsgpt.security.safe_url import (
ResponseTooLargeError,
UnsafeUserUrlError,
pinned_fetch_bytes,
@@ -11,18 +11,18 @@ import uuid
from datetime import datetime, timezone
from typing import Any, Dict, Optional
from application.agents.tools.base import Tool
from application.devices.broker import get_broker
from application.devices.denylist import check_denylist
from application.devices.normalizer import normalize_command
from application.storage.db.repositories.device_audit_log import (
from docsgpt.agents.tools.base import Tool
from docsgpt.devices.broker import get_broker
from docsgpt.devices.denylist import check_denylist
from docsgpt.devices.normalizer import normalize_command
from docsgpt.storage.db.repositories.device_audit_log import (
DeviceAuditLogRepository,
)
from application.storage.db.repositories.device_auto_approve_patterns import (
from docsgpt.storage.db.repositories.device_auto_approve_patterns import (
DeviceAutoApprovePatternsRepository,
)
from application.storage.db.repositories.devices import DevicesRepository
from application.storage.db.session import db_readonly, db_session
from docsgpt.storage.db.repositories.devices import DevicesRepository
from docsgpt.storage.db.session import db_readonly, db_session
logger = logging.getLogger(__name__)
@@ -7,16 +7,16 @@ import logging
from datetime import datetime, timezone
from typing import Any, Dict, List, Optional
from application.agents.scheduler_utils import (
from docsgpt.agents.scheduler_utils import (
ScheduleValidationError,
clamp_once_horizon,
parse_delay,
parse_run_at,
)
from application.core.settings import settings
from application.storage.db.base_repository import looks_like_uuid
from application.storage.db.repositories.schedules import SchedulesRepository
from application.storage.db.session import db_readonly, db_session
from docsgpt.core.settings import settings
from docsgpt.storage.db.base_repository import looks_like_uuid
from docsgpt.storage.db.repositories.schedules import SchedulesRepository
from docsgpt.storage.db.session import db_readonly, db_session
from .base import Tool
@@ -297,13 +297,13 @@ def _safe_default_allowlist(
chat tools (resolved against ``settings.DEFAULT_CHAT_TOOLS`` and the
user's ``tool_preferences.disabled_default_tools`` opt-outs).
"""
from application.agents.default_tools import (
from docsgpt.agents.default_tools import (
resolve_tool_by_id,
synthesized_default_tools,
)
from application.storage.db.repositories.agents import AgentsRepository
from application.storage.db.repositories.user_tools import UserToolsRepository
from application.storage.db.repositories.users import UsersRepository
from docsgpt.storage.db.repositories.agents import AgentsRepository
from docsgpt.storage.db.repositories.user_tools import UserToolsRepository
from docsgpt.storage.db.repositories.users import UsersRepository
def _is_safe(row: Dict[str, Any]) -> bool:
actions = row.get("actions") or []
File renamed without changes.
@@ -2,7 +2,7 @@ import logging
import requests
from application.agents.tools.base import Tool
from docsgpt.agents.tools.base import Tool
logger = logging.getLogger(__name__)
@@ -1,4 +1,4 @@
from application.agents.tools.base import Tool
from docsgpt.agents.tools.base import Tool
THINK_TOOL_ID = "think"
@@ -2,8 +2,8 @@ from typing import Any, Dict, List, Optional
import uuid
from .base import Tool
from application.storage.db.repositories.todos import TodosRepository
from application.storage.db.session import db_readonly, db_session
from docsgpt.storage.db.repositories.todos import TodosRepository
from docsgpt.storage.db.session import db_readonly, db_session
def _status_from_completed(completed: Any) -> str:
@@ -60,7 +60,7 @@ class TodoListTool(Tool):
return False
if tool_id.startswith("default_"):
return False
from application.storage.db.base_repository import looks_like_uuid
from docsgpt.storage.db.base_repository import looks_like_uuid
return looks_like_uuid(tool_id)
@@ -3,7 +3,7 @@ import inspect
import os
import pkgutil
from application.agents.tools.base import Tool
from docsgpt.agents.tools.base import Tool
class ToolManager:
@@ -17,7 +17,7 @@ class ToolManager:
for finder, name, ispkg in pkgutil.iter_modules([tools_dir]):
if name == "base" or name.startswith("__"):
continue
module = importlib.import_module(f"application.agents.tools.{name}")
module = importlib.import_module(f"docsgpt.agents.tools.{name}")
for member_name, obj in inspect.getmembers(module, inspect.isclass):
if issubclass(obj, Tool) and obj is not Tool and not obj.internal:
tool_config = self.config.get(name, {})
@@ -25,7 +25,7 @@ class ToolManager:
def load_tool(self, tool_name, tool_config, user_id=None):
self.config[tool_name] = tool_config
module = importlib.import_module(f"application.agents.tools.{tool_name}")
module = importlib.import_module(f"docsgpt.agents.tools.{tool_name}")
for member_name, obj in inspect.getmembers(module, inspect.isclass):
if issubclass(obj, Tool) and obj is not Tool:
if (
@@ -1,15 +1,15 @@
import logging
from typing import Any, Dict, List, Optional
from application.agents.tools.base import Tool
from application.agents.tools.path_utils import validate_tool_path
from application.storage.db.repositories.wiki_pages import (
from docsgpt.agents.tools.base import Tool
from docsgpt.agents.tools.path_utils import validate_tool_path
from docsgpt.storage.db.repositories.wiki_pages import (
WikiPageConflict,
WikiPagesRepository,
_content_hash,
rebuild_wiki_directory_structure,
)
from application.storage.db.session import db_readonly, db_session
from docsgpt.storage.db.session import db_readonly, db_session
logger = logging.getLogger(__name__)
@@ -207,7 +207,7 @@ class WikiTool(Tool):
idempotency key guards each edit independently and dedups broker
redeliveries without colliding across pages of the same source.
"""
from application.api.user.tasks import reembed_wiki_page
from docsgpt.api.user.tasks import reembed_wiki_page
reembed_wiki_page.delay(
self.source_id,
@@ -2,11 +2,11 @@ import logging
from datetime import datetime, timezone
from typing import Any, Dict, Generator, List, Optional, Tuple
from application.agents.base import BaseAgent
from application.guardrails.config import (
from docsgpt.agents.base import BaseAgent
from docsgpt.guardrails.config import (
DEFAULT_BLOCK_MESSAGE as GUARDRAIL_DEFAULT_MESSAGE,
)
from application.agents.workflows.schemas import (
from docsgpt.agents.workflows.schemas import (
ExecutionStatus,
Workflow,
WorkflowEdge,
@@ -14,16 +14,16 @@ from application.agents.workflows.schemas import (
WorkflowNode,
WorkflowRun,
)
from application.agents.workflows.workflow_engine import WorkflowEngine
from application.core.settings import settings
from application.logging import LogContext, log_activity
from application.sandbox.artifacts_capture import QuotaExceeded
from application.storage.db.base_repository import looks_like_uuid
from application.storage.db.repositories.workflow_edges import WorkflowEdgesRepository
from application.storage.db.repositories.workflow_nodes import WorkflowNodesRepository
from application.storage.db.repositories.workflow_runs import WorkflowRunsRepository
from application.storage.db.repositories.workflows import WorkflowsRepository
from application.storage.db.session import db_readonly, db_session
from docsgpt.agents.workflows.workflow_engine import WorkflowEngine
from docsgpt.core.settings import settings
from docsgpt.logging import LogContext, log_activity
from docsgpt.sandbox.artifacts_capture import QuotaExceeded
from docsgpt.storage.db.base_repository import looks_like_uuid
from docsgpt.storage.db.repositories.workflow_edges import WorkflowEdgesRepository
from docsgpt.storage.db.repositories.workflow_nodes import WorkflowNodesRepository
from docsgpt.storage.db.repositories.workflow_runs import WorkflowRunsRepository
from docsgpt.storage.db.repositories.workflows import WorkflowsRepository
from docsgpt.storage.db.session import db_readonly, db_session
logger = logging.getLogger(__name__)
@@ -327,8 +327,8 @@ class WorkflowAgent(BaseAgent):
# parent), so skip the bridge for unowned/draft ids.
if not persisted:
return [], []
from application.sandbox.artifacts_capture import persist_new_artifact
from application.storage.storage_creator import StorageCreator
from docsgpt.sandbox.artifacts_capture import persist_new_artifact
from docsgpt.storage.storage_creator import StorageCreator
storage = StorageCreator.get_storage()
max_bytes = int(getattr(settings, "ARTIFACT_MAX_BYTES", 0) or 0)
@@ -2,11 +2,11 @@
from typing import Dict, List, Optional, Type
from application.agents.agentic_agent import AgenticAgent
from application.agents.base import BaseAgent
from application.agents.classic_agent import ClassicAgent
from application.agents.research_agent import ResearchAgent
from application.agents.workflows.schemas import AgentType
from docsgpt.agents.agentic_agent import AgenticAgent
from docsgpt.agents.base import BaseAgent
from docsgpt.agents.classic_agent import ClassicAgent
from docsgpt.agents.research_agent import ResearchAgent
from docsgpt.agents.workflows.schemas import AgentType
class _WorkflowNodeMixin:
File renamed without changes.
@@ -6,9 +6,9 @@ import uuid
from datetime import datetime, timezone
from typing import Any, Dict, Generator, List, Optional, TYPE_CHECKING
from application.agents.workflows.cel_evaluator import CelEvaluationError, evaluate_cel
from application.agents.workflows.node_agent import WorkflowNodeAgentFactory
from application.agents.workflows.schemas import (
from docsgpt.agents.workflows.cel_evaluator import CelEvaluationError, evaluate_cel
from docsgpt.agents.workflows.node_agent import WorkflowNodeAgentFactory
from docsgpt.agents.workflows.schemas import (
AgentNodeConfig,
AgentType,
CodeNodeConfig,
@@ -19,13 +19,13 @@ from application.agents.workflows.schemas import (
WorkflowGraph,
WorkflowNode,
)
from application.core.json_schema_utils import (
from docsgpt.core.json_schema_utils import (
JsonSchemaValidationError,
normalize_json_schema_payload,
)
from application.error import sanitize_api_error
from application.templates.namespaces import NamespaceManager
from application.templates.template_engine import TemplateEngine, TemplateRenderError
from docsgpt.error import sanitize_api_error
from docsgpt.templates.namespaces import NamespaceManager
from docsgpt.templates.template_engine import TemplateEngine, TemplateRenderError
try:
import jsonschema
@@ -33,7 +33,7 @@ except ImportError: # pragma: no cover - optional dependency in some deployment
jsonschema = None
if TYPE_CHECKING:
from application.agents.base import BaseAgent
from docsgpt.agents.base import BaseAgent
logger = logging.getLogger(__name__)
StateValue = Any
@@ -101,7 +101,7 @@ class WorkflowEngine:
# node and agent-node tool in this run, so it is torn down exactly once
# here rather than per node. peek_manager() never builds the manager, so
# a run that never opened a session closes nothing.
from application.sandbox.sandbox_creator import SandboxCreator
from docsgpt.sandbox.sandbox_creator import SandboxCreator
mgr = SandboxCreator.peek_manager()
if mgr is not None:
@@ -312,13 +312,13 @@ class WorkflowEngine:
def _execute_agent_node(
self, node: WorkflowNode
) -> Generator[Dict[str, str], None, None]:
from application.core.model_utils import (
from docsgpt.core.model_utils import (
get_api_key_for_provider,
get_model_capabilities,
resolve_dispatch_provider,
)
from application.api.answer.services.prompt_renderer import (
from docsgpt.api.answer.services.prompt_renderer import (
prompt_embeds_documents as _prompt_embeds_documents,
)
@@ -512,8 +512,8 @@ class WorkflowEngine:
self, node: WorkflowNode
) -> Generator[Dict[str, str], None, None]:
"""Run code in the run-scoped sandbox, persist produced files, and write an artifact reference."""
from application.sandbox.artifacts_capture import capture_artifacts, snapshot_signatures
from application.sandbox.sandbox_creator import SandboxCreator
from docsgpt.sandbox.artifacts_capture import capture_artifacts, snapshot_signatures
from docsgpt.sandbox.sandbox_creator import SandboxCreator
config = CodeNodeConfig(**node.config.get("config", node.config))
code = config.code or ""
@@ -627,13 +627,13 @@ class WorkflowEngine:
self, manager: Any, session_id: str, inputs: List[str], user_id: str
) -> List[str]:
"""Stage referenced input artifacts (run-scoped, never cross-tenant) into the workspace."""
from application.agents.tools.artifact_ref import resolve_artifact_id
from application.core.settings import settings
from application.sandbox.artifacts_capture import unique_input_path
from application.storage.db.repositories.artifacts import ArtifactsRepository
from application.storage.db.session import db_readonly
from application.storage.storage_creator import StorageCreator
from application.utils import safe_filename
from docsgpt.agents.tools.artifact_ref import resolve_artifact_id
from docsgpt.core.settings import settings
from docsgpt.sandbox.artifacts_capture import unique_input_path
from docsgpt.storage.db.repositories.artifacts import ArtifactsRepository
from docsgpt.storage.db.session import db_readonly
from docsgpt.storage.storage_creator import StorageCreator
from docsgpt.utils import safe_filename
loaded: List[str] = []
raw_ids = self._resolve_input_artifact_ids(inputs)
@@ -693,9 +693,9 @@ class WorkflowEngine:
selection, so it never widens per-node document access. Best-effort:
a resolution failure drops the manifest, never the node.
"""
from application.agents.tools.artifact_ref import make_ref, resolve_artifact_id
from application.storage.db.repositories.artifacts import ArtifactsRepository
from application.storage.db.session import db_readonly
from docsgpt.agents.tools.artifact_ref import make_ref, resolve_artifact_id
from docsgpt.storage.db.repositories.artifacts import ArtifactsRepository
from docsgpt.storage.db.session import db_readonly
try:
raw_ids = self._resolve_input_artifact_ids(node_config.input_documents)
@@ -738,10 +738,10 @@ class WorkflowEngine:
supported_types: List[str],
) -> List[Dict[str, Any]]:
"""Resolve a node's selected documents to native/extracted attachment dicts for its LLM."""
from application.agents.tools.artifact_ref import resolve_artifact_id
from application.core.settings import settings
from application.storage.db.repositories.artifacts import ArtifactsRepository
from application.storage.db.session import db_readonly
from docsgpt.agents.tools.artifact_ref import resolve_artifact_id
from docsgpt.core.settings import settings
from docsgpt.storage.db.repositories.artifacts import ArtifactsRepository
from docsgpt.storage.db.session import db_readonly
raw_ids = self._resolve_input_artifact_ids(node_config.input_documents)
if not raw_ids:
@@ -909,8 +909,8 @@ class WorkflowEngine:
deadline: ``time.monotonic()`` value past which no further blocking
parse may run, shared across the node's documents.
"""
from application.parser.document_reader import truncate_text_head_tail
from application.storage.storage_creator import StorageCreator
from docsgpt.parser.document_reader import truncate_text_head_tail
from docsgpt.storage.storage_creator import StorageCreator
# An uploaded chat attachment was already parsed (and possibly OCR'd) when it
# was stored, so re-parsing it here would repeat the dominant cost of the run
@@ -965,12 +965,12 @@ class WorkflowEngine:
"""
from celery.exceptions import TimeoutError as CeleryTimeoutError
from application.api.user.tasks import (
from docsgpt.api.user.tasks import (
parse_document,
parse_task_time_limits,
parse_timeout_for_size,
)
from application.core.settings import settings
from docsgpt.core.settings import settings
user_id = self._resolve_user_id()
if not user_id:
@@ -999,7 +999,7 @@ class WorkflowEngine:
# default ``disable_sync_subtasks=True`` makes ``get()`` raise RuntimeError
# ("Never call result.get() within a task!"). The dedicated parsing queue +
# separate workers already avoid the real self-deadlock, so opt out
# explicitly (mirrors application/agents/tools/read_document.py).
# explicitly (mirrors docsgpt/agents/tools/read_document.py).
result = async_result.get(timeout=timeout, disable_sync_subtasks=False)
except (CeleryTimeoutError, TimeoutError):
logger.warning("Workflow node: document parse timed out for %s", artifact_id)
@@ -1082,7 +1082,7 @@ class WorkflowEngine:
def _resolve_code_timeout(self, requested: Optional[int]) -> float:
"""Return the stricter of the node's requested timeout and the sandbox cap."""
from application.core.settings import settings
from docsgpt.core.settings import settings
cap = float(getattr(settings, "SANDBOX_EXEC_TIMEOUT", 60))
if requested is None:
@@ -1331,8 +1331,8 @@ class WorkflowEngine:
logger.warning("Workflow node sources dropped: no owner to authorize.")
return []
from application.api.user.team_sharing import can_access
from application.storage.db.session import db_readonly
from docsgpt.api.user.team_sharing import can_access
from docsgpt.storage.db.session import db_readonly
allowed = []
try:
@@ -1359,7 +1359,7 @@ class WorkflowEngine:
Returns:
list: Retrieved documents, empty when there was nothing to fetch.
"""
from application.retriever.retriever_creator import RetrieverCreator
from docsgpt.retriever.retriever_creator import RetrieverCreator
query = self.state.get("query", "")
if not query:
@@ -1,11 +1,11 @@
# Alembic configuration for the DocsGPT user-data Postgres database.
#
# The SQLAlchemy URL is deliberately NOT set here — env.py reads it from
# ``application.core.settings.settings.POSTGRES_URI`` so the same config
# ``docsgpt.core.settings.settings.POSTGRES_URI`` so the same config
# source serves the running app and migrations. To run from the project
# root::
#
# alembic -c application/alembic.ini upgrade head
# alembic -c docsgpt/alembic.ini upgrade head
[alembic]
script_location = %(here)s/alembic
@@ -1,6 +1,6 @@
"""Alembic environment for the DocsGPT user-data Postgres database.
The URL is pulled from ``application.core.settings`` rather than
The URL is pulled from ``docsgpt.core.settings`` rather than
``alembic.ini`` so that a single ``POSTGRES_URI`` env var drives both the
running app and ``alembic`` CLI invocations.
"""
@@ -10,7 +10,7 @@ from logging.config import fileConfig
from pathlib import Path
# Make the project root importable regardless of cwd. env.py lives at
# <repo>/application/alembic/env.py, so parents[2] is the repo root.
# <repo>/docsgpt/alembic/env.py, so parents[2] is the repo root.
_PROJECT_ROOT = Path(__file__).resolve().parents[2]
if str(_PROJECT_ROOT) not in sys.path:
sys.path.insert(0, str(_PROJECT_ROOT))
@@ -18,8 +18,8 @@ if str(_PROJECT_ROOT) not in sys.path:
from alembic import context # noqa: E402
from sqlalchemy import engine_from_config, pool # noqa: E402
from application.core.settings import settings # noqa: E402
from application.storage.db.models import metadata as target_metadata # noqa: E402
from docsgpt.core.settings import settings # noqa: E402
from docsgpt.storage.db.models import metadata as target_metadata # noqa: E402
config = context.config
File renamed without changes.
Loaded 100 of 1006 files, more files were not shown because too many files have changed in this diff. Show more