mirror of
https://github.com/tiennm99/DocsGPT.git
synced 2026-10-13 02:14:23 +00:00
Merge branch 'arc53:main' into main
This commit is contained in:
1006 files changed
+21714
-10075
No files matched your search
@@ -19,17 +19,17 @@ Run the full app under uvicorn (serves `/mcp` and the async SSE reconnect
|
||||
routes, and matches production):
|
||||
|
||||
```bash
|
||||
uvicorn application.asgi:asgi_app --host 0.0.0.0 --port 7091 --reload
|
||||
uvicorn docsgpt.asgi:asgi_app --host 0.0.0.0 --port 7091 --reload
|
||||
```
|
||||
|
||||
`flask --app application/app.py run --host=0.0.0.0 --port=7091` is faster but
|
||||
`flask --app docsgpt/app.py run --host=0.0.0.0 --port=7091` is faster but
|
||||
serves only the WSGI Flask app — it omits `/mcp` and the reconnect reader
|
||||
`GET /api/messages/<id>/events`, so a dropped stream won't auto-resume.
|
||||
|
||||
### Celery (Task Queue)
|
||||
|
||||
```bash
|
||||
celery -A application.app.celery worker -l INFO -Q docsgpt,parsing,embeddings
|
||||
celery -A docsgpt.app.celery worker -l INFO -Q docsgpt,parsing,embeddings
|
||||
```
|
||||
|
||||
The `parsing` queue serves document parsing (the `read_document` tool / workflow
|
||||
|
||||
@@ -22,8 +22,8 @@ fi
|
||||
|
||||
|
||||
# The embedding model is fetched on first use and cached, so nothing to download
|
||||
# here. For an offline container, run `python -m application.scripts.prefetch_models`
|
||||
# here. For an offline container, run `python -m docsgpt.scripts.prefetch_models`
|
||||
# after the install below.
|
||||
pip install -r application/requirements.txt
|
||||
pip install -r docsgpt/requirements.txt
|
||||
cd frontend
|
||||
npm install --include=dev
|
||||
@@ -0,0 +1,25 @@
|
||||
# Build context for docsgpt/Dockerfile is the repository root, so the image can
|
||||
# carry the `application` import alias next to the `docsgpt` package. Allow only
|
||||
# what the image needs; everything else (frontend, docs, tests, venvs) stays out.
|
||||
*
|
||||
!docsgpt/
|
||||
# Only the alias file: an upgraded checkout may still hold gitignored
|
||||
# application/{inputs,indexes,vectors,.env} from the old layout.
|
||||
!application/__init__.py
|
||||
|
||||
# Inside the package: caches, local runtime data and secrets never ship.
|
||||
**/__pycache__/
|
||||
**/*.py[cod]
|
||||
docsgpt/.pytest_cache/
|
||||
docsgpt/.ruff_cache/
|
||||
docsgpt/.coverage
|
||||
docsgpt/htmlcov/
|
||||
docsgpt/*.log
|
||||
docsgpt/indexes/
|
||||
docsgpt/inputs/
|
||||
docsgpt/vectors/
|
||||
docsgpt/*.faiss
|
||||
docsgpt/*.pkl
|
||||
docsgpt/.env
|
||||
docsgpt/.env.*
|
||||
docsgpt/Dockerfile
|
||||
+1
-1
@@ -21,7 +21,7 @@ INTERNAL_KEY=<internal key for worker-to-backend authentication>
|
||||
# searching a different vector space than the stored vectors -- which fails
|
||||
# silently, because both models are 768-dimensional. To switch, set it and then
|
||||
# run:
|
||||
# python -m application.scripts.reembed
|
||||
# python -m docsgpt.scripts.reembed
|
||||
# EMBEDDINGS_NAME=ibm-granite/granite-embedding-311m-multilingual-r2
|
||||
|
||||
# Remote Embeddings (Optional - for using a remote embeddings API instead of
|
||||
|
||||
@@ -10,7 +10,7 @@ DocsGPT ingests content (files/URLs/connectors), indexes it, and answers queries
|
||||
|
||||
Core components:
|
||||
- Backend API (`application/`)
|
||||
- Workers/ingestion (`application/worker.py` and related modules)
|
||||
- Workers/ingestion (`docsgpt/worker.py` and related modules)
|
||||
- Datastores (MongoDB/Redis/vector stores)
|
||||
- Frontend (`frontend/`)
|
||||
- Optional extensions/integrations (`extensions/`)
|
||||
|
||||
@@ -6,7 +6,7 @@
|
||||
version: 2
|
||||
updates:
|
||||
- package-ecosystem: "pip" # See documentation for possible values
|
||||
directory: "/application" # Location of package manifests
|
||||
directory: "/docsgpt" # Location of package manifests
|
||||
schedule:
|
||||
interval: "daily"
|
||||
- package-ecosystem: "npm" # See documentation for possible values
|
||||
|
||||
+1
-1
@@ -8,7 +8,7 @@ github:
|
||||
|
||||
application:
|
||||
- changed-files:
|
||||
- any-glob-to-any-file: 'application/**/*'
|
||||
- any-glob-to-any-file: 'docsgpt/**/*'
|
||||
|
||||
docs:
|
||||
- changed-files:
|
||||
|
||||
@@ -4,7 +4,7 @@ on:
|
||||
push:
|
||||
branches: [main]
|
||||
paths:
|
||||
- 'application/version.py'
|
||||
- 'docsgpt/version.py'
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
@@ -23,12 +23,12 @@ jobs:
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Read version from application/version.py
|
||||
- name: Read version from docsgpt/version.py
|
||||
id: ver
|
||||
run: |
|
||||
VERSION=$(python3 -c "g={}; exec(open('application/version.py').read(), g); print(g['__version__'])")
|
||||
VERSION=$(python3 -c "g={}; exec(open('docsgpt/version.py').read(), g); print(g['__version__'])")
|
||||
if [ -z "$VERSION" ]; then
|
||||
echo "::error::Could not read __version__ from application/version.py"
|
||||
echo "::error::Could not read __version__ from docsgpt/version.py"
|
||||
exit 1
|
||||
fi
|
||||
echo "version=$VERSION" >> "$GITHUB_OUTPUT"
|
||||
|
||||
@@ -28,13 +28,13 @@ jobs:
|
||||
run: |
|
||||
python -m pip install --upgrade pip
|
||||
pip install bandit # Bandit is needed for this action
|
||||
if [ -f application/requirements.txt ]; then pip install -r application/requirements.txt; fi
|
||||
if [ -f docsgpt/requirements.txt ]; then pip install -r docsgpt/requirements.txt; fi
|
||||
|
||||
- name: Run Bandit scan
|
||||
uses: PyCQA/bandit-action@v1
|
||||
with:
|
||||
severity: medium
|
||||
confidence: medium
|
||||
targets: application/
|
||||
targets: docsgpt/
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
+83
-45
@@ -7,106 +7,144 @@ on:
|
||||
jobs:
|
||||
build:
|
||||
if: github.repository == 'arc53/DocsGPT'
|
||||
# Publishing jobs run in a GitHub Actions environment so the registry
|
||||
# credentials can be scoped to it and protection rules (required reviewers,
|
||||
# branch restrictions) applied in the repository settings.
|
||||
environment: docker-hub
|
||||
env:
|
||||
# Public namespace the compose files pull from; the login secret only
|
||||
# authenticates the push.
|
||||
DOCKERHUB_NAMESPACE: arc53
|
||||
strategy:
|
||||
matrix:
|
||||
include:
|
||||
- platform: linux/amd64
|
||||
runner: ubuntu-latest
|
||||
suffix: amd64
|
||||
- platform: linux/arm64
|
||||
runner: ubuntu-24.04-arm
|
||||
suffix: arm64
|
||||
runs-on: ${{ matrix.runner }}
|
||||
platform: [linux/amd64, linux/arm64]
|
||||
# "" is the slim default image; "-docling" bakes the docling parser
|
||||
# engine, its models and tesseract in (OCR-ready).
|
||||
variant: ["", "-docling"]
|
||||
runs-on: ${{ matrix.platform == 'linux/arm64' && 'ubuntu-24.04-arm' || 'ubuntu-latest' }}
|
||||
permissions:
|
||||
contents: read
|
||||
packages: write
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Set up QEMU # Only needed for emulation, not for native arm64 builds
|
||||
if: matrix.platform == 'linux/arm64'
|
||||
uses: docker/setup-qemu-action@v3
|
||||
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v3
|
||||
uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3.12.0
|
||||
with:
|
||||
driver: docker-container
|
||||
install: true
|
||||
|
||||
- name: Login to DockerHub
|
||||
uses: docker/login-action@v3
|
||||
uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3.7.0
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||
|
||||
- name: Login to ghcr.io
|
||||
uses: docker/login-action@v3
|
||||
uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3.7.0
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ github.repository_owner }}
|
||||
password: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
- name: Build and push platform-specific images
|
||||
uses: docker/build-push-action@v6
|
||||
- name: Image metadata (OCI labels)
|
||||
id: meta
|
||||
uses: docker/metadata-action@c299e40c65443455700f0fdfc63efafe5b349051 # v5.10.0
|
||||
with:
|
||||
file: './application/Dockerfile'
|
||||
images: |
|
||||
${{ env.DOCKERHUB_NAMESPACE }}/docsgpt
|
||||
ghcr.io/${{ github.repository_owner }}/docsgpt
|
||||
labels: |
|
||||
org.opencontainers.image.title=DocsGPT${{ matrix.variant }}
|
||||
org.opencontainers.image.version=${{ github.event.release.tag_name }}
|
||||
|
||||
- name: Build and push platform-specific images
|
||||
uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6.19.2
|
||||
with:
|
||||
file: './docsgpt/Dockerfile'
|
||||
platforms: ${{ matrix.platform }}
|
||||
context: ./application
|
||||
context: .
|
||||
push: true
|
||||
build-args: |
|
||||
EXTRAS=${{ matrix.variant == '-docling' && 'docling' || '' }}
|
||||
INSTALL_TESSERACT=${{ matrix.variant == '-docling' && 'true' || 'false' }}
|
||||
tags: |
|
||||
${{ secrets.DOCKER_USERNAME }}/docsgpt:${{ github.event.release.tag_name }}-${{ matrix.suffix }}
|
||||
ghcr.io/${{ github.repository_owner }}/docsgpt:${{ github.event.release.tag_name }}-${{ matrix.suffix }}
|
||||
${{ env.DOCKERHUB_NAMESPACE }}/docsgpt:${{ github.event.release.tag_name }}${{ matrix.variant }}-${{ matrix.platform == 'linux/arm64' && 'arm64' || 'amd64' }}
|
||||
ghcr.io/${{ github.repository_owner }}/docsgpt:${{ github.event.release.tag_name }}${{ matrix.variant }}-${{ matrix.platform == 'linux/arm64' && 'arm64' || 'amd64' }}
|
||||
labels: ${{ steps.meta.outputs.labels }}
|
||||
provenance: false
|
||||
sbom: false
|
||||
cache-from: type=registry,ref=${{ secrets.DOCKER_USERNAME }}/docsgpt:latest
|
||||
cache-from: type=registry,ref=${{ env.DOCKERHUB_NAMESPACE }}/docsgpt:latest${{ matrix.variant }}
|
||||
cache-to: type=inline
|
||||
|
||||
manifest:
|
||||
if: github.repository == 'arc53/DocsGPT'
|
||||
# Publishing jobs run in a GitHub Actions environment so the registry
|
||||
# credentials can be scoped to it and protection rules (required reviewers,
|
||||
# branch restrictions) applied in the repository settings.
|
||||
environment: docker-hub
|
||||
env:
|
||||
# Public namespace the compose files pull from; the login secret only
|
||||
# authenticates the push.
|
||||
DOCKERHUB_NAMESPACE: arc53
|
||||
needs: build
|
||||
strategy:
|
||||
matrix:
|
||||
variant: ["", "-docling"]
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
packages: write
|
||||
steps:
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v3
|
||||
uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3.12.0
|
||||
with:
|
||||
driver: docker-container
|
||||
install: true
|
||||
|
||||
- name: Login to DockerHub
|
||||
uses: docker/login-action@v3
|
||||
uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3.7.0
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||
|
||||
- name: Login to ghcr.io
|
||||
uses: docker/login-action@v3
|
||||
uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3.7.0
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ github.repository_owner }}
|
||||
password: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
- name: Create and push manifest for DockerHub
|
||||
- name: Create and push multi-arch manifests
|
||||
env:
|
||||
TAG: ${{ github.event.release.tag_name }}${{ matrix.variant }}
|
||||
LATEST: latest${{ matrix.variant }}
|
||||
run: |
|
||||
set -e
|
||||
docker manifest create ${{ secrets.DOCKER_USERNAME }}/docsgpt:${{ github.event.release.tag_name }} \
|
||||
--amend ${{ secrets.DOCKER_USERNAME }}/docsgpt:${{ github.event.release.tag_name }}-amd64 \
|
||||
--amend ${{ secrets.DOCKER_USERNAME }}/docsgpt:${{ github.event.release.tag_name }}-arm64
|
||||
docker manifest push ${{ secrets.DOCKER_USERNAME }}/docsgpt:${{ github.event.release.tag_name }}
|
||||
docker manifest create ${{ secrets.DOCKER_USERNAME }}/docsgpt:latest \
|
||||
--amend ${{ secrets.DOCKER_USERNAME }}/docsgpt:${{ github.event.release.tag_name }}-amd64 \
|
||||
--amend ${{ secrets.DOCKER_USERNAME }}/docsgpt:${{ github.event.release.tag_name }}-arm64
|
||||
docker manifest push ${{ secrets.DOCKER_USERNAME }}/docsgpt:latest
|
||||
for repo in "$DOCKERHUB_NAMESPACE/docsgpt" "ghcr.io/${{ github.repository_owner }}/docsgpt"; do
|
||||
for name in "$TAG" "$LATEST"; do
|
||||
docker manifest create "$repo:$name" \
|
||||
--amend "$repo:$TAG-amd64" \
|
||||
--amend "$repo:$TAG-arm64"
|
||||
docker manifest push "$repo:$name"
|
||||
done
|
||||
done
|
||||
|
||||
- name: Create and push manifest for ghcr.io
|
||||
release-assets:
|
||||
if: github.repository == 'arc53/DocsGPT'
|
||||
needs: manifest
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: write
|
||||
steps:
|
||||
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Attach the standalone compose file to the release
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
TAG: ${{ github.event.release.tag_name }}
|
||||
run: |
|
||||
set -e
|
||||
docker manifest create ghcr.io/${{ github.repository_owner }}/docsgpt:${{ github.event.release.tag_name }} \
|
||||
--amend ghcr.io/${{ github.repository_owner }}/docsgpt:${{ github.event.release.tag_name }}-amd64 \
|
||||
--amend ghcr.io/${{ github.repository_owner }}/docsgpt:${{ github.event.release.tag_name }}-arm64
|
||||
docker manifest push ghcr.io/${{ github.repository_owner }}/docsgpt:${{ github.event.release.tag_name }}
|
||||
docker manifest create ghcr.io/${{ github.repository_owner }}/docsgpt:latest \
|
||||
--amend ghcr.io/${{ github.repository_owner }}/docsgpt:${{ github.event.release.tag_name }}-amd64 \
|
||||
--amend ghcr.io/${{ github.repository_owner }}/docsgpt:${{ github.event.release.tag_name }}-arm64
|
||||
docker manifest push ghcr.io/${{ github.repository_owner }}/docsgpt:latest
|
||||
gh release upload "$TAG" deployment/docker-compose-standalone.yaml --clobber
|
||||
@@ -9,92 +9,123 @@ on:
|
||||
jobs:
|
||||
build:
|
||||
if: github.repository == 'arc53/DocsGPT'
|
||||
# Publishing jobs run in a GitHub Actions environment so the registry
|
||||
# credentials can be scoped to it and protection rules (required reviewers,
|
||||
# branch restrictions) applied in the repository settings.
|
||||
environment: docker-hub
|
||||
env:
|
||||
# Public namespace the compose files pull from; the login secret only
|
||||
# authenticates the push.
|
||||
DOCKERHUB_NAMESPACE: arc53
|
||||
strategy:
|
||||
matrix:
|
||||
include:
|
||||
- platform: linux/amd64
|
||||
runner: ubuntu-latest
|
||||
suffix: amd64
|
||||
- platform: linux/arm64
|
||||
runner: ubuntu-24.04-arm
|
||||
suffix: arm64
|
||||
runs-on: ${{ matrix.runner }}
|
||||
platform: [linux/amd64, linux/arm64]
|
||||
# "" is the slim default image; "-docling" bakes the docling parser
|
||||
# engine, its models and tesseract in (OCR-ready).
|
||||
variant: ["", "-docling"]
|
||||
runs-on: ${{ matrix.platform == 'linux/arm64' && 'ubuntu-24.04-arm' || 'ubuntu-latest' }}
|
||||
permissions:
|
||||
contents: read
|
||||
packages: write
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v3
|
||||
uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3.12.0
|
||||
with:
|
||||
driver: docker-container
|
||||
install: true
|
||||
|
||||
- name: Login to DockerHub
|
||||
uses: docker/login-action@v3
|
||||
uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3.7.0
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||
|
||||
|
||||
- name: Login to ghcr.io
|
||||
uses: docker/login-action@v3
|
||||
uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3.7.0
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ github.repository_owner }}
|
||||
password: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
- name: Build and push platform-specific images
|
||||
uses: docker/build-push-action@v6
|
||||
- name: Image metadata (OCI labels)
|
||||
id: meta
|
||||
uses: docker/metadata-action@c299e40c65443455700f0fdfc63efafe5b349051 # v5.10.0
|
||||
with:
|
||||
file: './application/Dockerfile'
|
||||
images: |
|
||||
${{ env.DOCKERHUB_NAMESPACE }}/docsgpt
|
||||
ghcr.io/${{ github.repository_owner }}/docsgpt
|
||||
labels: |
|
||||
org.opencontainers.image.title=DocsGPT${{ matrix.variant }}
|
||||
org.opencontainers.image.version=develop
|
||||
|
||||
- name: Build and push platform-specific images
|
||||
uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6.19.2
|
||||
with:
|
||||
file: './docsgpt/Dockerfile'
|
||||
platforms: ${{ matrix.platform }}
|
||||
context: ./application
|
||||
context: .
|
||||
push: true
|
||||
build-args: |
|
||||
EXTRAS=${{ matrix.variant == '-docling' && 'docling' || '' }}
|
||||
INSTALL_TESSERACT=${{ matrix.variant == '-docling' && 'true' || 'false' }}
|
||||
tags: |
|
||||
${{ secrets.DOCKER_USERNAME }}/docsgpt:develop-${{ matrix.suffix }}
|
||||
ghcr.io/${{ github.repository_owner }}/docsgpt:develop-${{ matrix.suffix }}
|
||||
${{ env.DOCKERHUB_NAMESPACE }}/docsgpt:develop${{ matrix.variant }}-${{ matrix.platform == 'linux/arm64' && 'arm64' || 'amd64' }}
|
||||
ghcr.io/${{ github.repository_owner }}/docsgpt:develop${{ matrix.variant }}-${{ matrix.platform == 'linux/arm64' && 'arm64' || 'amd64' }}
|
||||
labels: ${{ steps.meta.outputs.labels }}
|
||||
provenance: false
|
||||
sbom: false
|
||||
cache-from: type=registry,ref=${{ secrets.DOCKER_USERNAME }}/docsgpt:develop
|
||||
cache-from: type=registry,ref=${{ env.DOCKERHUB_NAMESPACE }}/docsgpt:develop${{ matrix.variant }}
|
||||
cache-to: type=inline
|
||||
|
||||
manifest:
|
||||
if: github.repository == 'arc53/DocsGPT'
|
||||
# Publishing jobs run in a GitHub Actions environment so the registry
|
||||
# credentials can be scoped to it and protection rules (required reviewers,
|
||||
# branch restrictions) applied in the repository settings.
|
||||
environment: docker-hub
|
||||
env:
|
||||
# Public namespace the compose files pull from; the login secret only
|
||||
# authenticates the push.
|
||||
DOCKERHUB_NAMESPACE: arc53
|
||||
needs: build
|
||||
strategy:
|
||||
matrix:
|
||||
variant: ["", "-docling"]
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
packages: write
|
||||
steps:
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v3
|
||||
uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3.12.0
|
||||
with:
|
||||
driver: docker-container
|
||||
install: true
|
||||
|
||||
- name: Login to DockerHub
|
||||
uses: docker/login-action@v3
|
||||
uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3.7.0
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||
|
||||
|
||||
- name: Login to ghcr.io
|
||||
uses: docker/login-action@v3
|
||||
uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3.7.0
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ github.repository_owner }}
|
||||
password: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
- name: Create and push manifest for DockerHub
|
||||
run: |
|
||||
docker manifest create ${{ secrets.DOCKER_USERNAME }}/docsgpt:develop \
|
||||
--amend ${{ secrets.DOCKER_USERNAME }}/docsgpt:develop-amd64 \
|
||||
--amend ${{ secrets.DOCKER_USERNAME }}/docsgpt:develop-arm64
|
||||
docker manifest push ${{ secrets.DOCKER_USERNAME }}/docsgpt:develop
|
||||
|
||||
- name: Create and push manifest for ghcr.io
|
||||
- name: Create and push multi-arch manifests
|
||||
env:
|
||||
TAG: develop${{ matrix.variant }}
|
||||
run: |
|
||||
docker manifest create ghcr.io/${{ github.repository_owner }}/docsgpt:develop \
|
||||
--amend ghcr.io/${{ github.repository_owner }}/docsgpt:develop-amd64 \
|
||||
--amend ghcr.io/${{ github.repository_owner }}/docsgpt:develop-arm64
|
||||
docker manifest push ghcr.io/${{ github.repository_owner }}/docsgpt:develop
|
||||
set -e
|
||||
for repo in "$DOCKERHUB_NAMESPACE/docsgpt" "ghcr.io/${{ github.repository_owner }}/docsgpt"; do
|
||||
docker manifest create "$repo:$TAG" \
|
||||
--amend "$repo:$TAG-amd64" \
|
||||
--amend "$repo:$TAG-arm64"
|
||||
docker manifest push "$repo:$TAG"
|
||||
done
|
||||
@@ -0,0 +1,67 @@
|
||||
name: Verify the Docker image works offline
|
||||
|
||||
# Builds the backend image and runs its offline check with networking off, so
|
||||
# a change that reintroduces a first-request download (a tokenizer, tiktoken's
|
||||
# encoding, an embedding model) fails here instead of in an air-gapped install.
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
pull_request:
|
||||
paths:
|
||||
- 'docsgpt/Dockerfile'
|
||||
- '.dockerignore'
|
||||
- 'application/**'
|
||||
- 'docsgpt/requirements*.txt'
|
||||
- 'docsgpt/scripts/prefetch_models.py'
|
||||
- 'docsgpt/scripts/verify_offline.py'
|
||||
- 'docsgpt/vectorstore/model_registry.py'
|
||||
- 'docsgpt/parser/tokenization.py'
|
||||
- 'docsgpt/vectorstore/embeddings_local.py'
|
||||
- '.github/workflows/docker-image-verify.yml'
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
verify:
|
||||
strategy:
|
||||
matrix:
|
||||
# "" is the slim default; "-docling" bakes docling, its models and
|
||||
# tesseract in, so the conversion check in verify_offline runs too.
|
||||
variant: ["", "-docling"]
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3.12.0
|
||||
|
||||
- name: Build the image
|
||||
uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6.19.2
|
||||
with:
|
||||
file: ./docsgpt/Dockerfile
|
||||
context: .
|
||||
platforms: linux/amd64
|
||||
load: true
|
||||
tags: docsgpt:verify${{ matrix.variant }}
|
||||
build-args: |
|
||||
EXTRAS=${{ matrix.variant == '-docling' && 'docling' || '' }}
|
||||
INSTALL_TESSERACT=${{ matrix.variant == '-docling' && 'true' || 'false' }}
|
||||
cache-from: type=gha,scope=verify${{ matrix.variant }}
|
||||
cache-to: type=gha,mode=max,scope=verify${{ matrix.variant }}
|
||||
|
||||
- name: Image size
|
||||
env:
|
||||
IMAGE: docsgpt:verify${{ matrix.variant }}
|
||||
run: |
|
||||
docker image inspect "$IMAGE" --format '{{.Size}}' | awk '{printf "uncompressed: %.2f GB\n", $1/1e9}'
|
||||
docker history "$IMAGE" --format '{{.Size}}\t{{.CreatedBy}}' | head -20
|
||||
|
||||
- name: Offline verification (no network)
|
||||
env:
|
||||
IMAGE: docsgpt:verify${{ matrix.variant }}
|
||||
run: |
|
||||
docker run --rm --network none "$IMAGE" \
|
||||
python -m docsgpt.scripts.verify_offline
|
||||
@@ -20,3 +20,20 @@ jobs:
|
||||
uses: chartboost/ruff-action@v1
|
||||
with:
|
||||
version: 0.14.10
|
||||
|
||||
requirements-in-sync:
|
||||
# application/requirements*.txt are exported from uv.lock; fail when a
|
||||
# change to pyproject.toml or uv.lock was not re-exported.
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6.8.0
|
||||
|
||||
- name: Re-export and diff
|
||||
run: |
|
||||
uv lock --check
|
||||
bash scripts/export_requirements.sh
|
||||
git diff --exit-code -- docsgpt/requirements.txt docsgpt/requirements-docling.txt docsgpt/requirements-milvus.txt
|
||||
@@ -20,13 +20,13 @@ jobs:
|
||||
- name: Install dependencies
|
||||
run: |
|
||||
python -m pip install --upgrade pip
|
||||
cd application
|
||||
cd docsgpt
|
||||
if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
|
||||
cd ../tests
|
||||
if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
|
||||
- name: Test with pytest and generate coverage report
|
||||
run: |
|
||||
python -m pytest -n auto --cov=application --cov-report=xml --cov-report=term-missing
|
||||
python -m pytest -n auto --cov=docsgpt --cov-report=xml --cov-report=term-missing
|
||||
- name: Upload coverage reports to Codecov
|
||||
if: github.event_name == 'pull_request' && matrix.python-version == '3.12'
|
||||
uses: codecov/codecov-action@v5
|
||||
|
||||
+2
-1
@@ -179,6 +179,7 @@ frontend/*.njsproj
|
||||
frontend/*.sln
|
||||
frontend/*.sw?
|
||||
|
||||
docsgpt/vectors/
|
||||
application/vectors/
|
||||
|
||||
**/inputs
|
||||
@@ -202,6 +203,6 @@ tests/e2e/node_modules/
|
||||
tests/e2e/playwright-report/
|
||||
tests/e2e/test-results/
|
||||
tests/e2e/.e2e-last-run.json
|
||||
application/core/models/foundry_test_internal.yaml
|
||||
docsgpt/core/models/foundry_test_internal.yaml
|
||||
bench/
|
||||
/benchmark/
|
||||
Vendored
+2
-2
@@ -14,7 +14,7 @@
|
||||
"request": "launch",
|
||||
"module": "flask",
|
||||
"env": {
|
||||
"FLASK_APP": "application/app.py",
|
||||
"FLASK_APP": "docsgpt/app.py",
|
||||
"PYTHONPATH": "${workspaceFolder}",
|
||||
"FLASK_ENV": "development",
|
||||
"FLASK_DEBUG": "1",
|
||||
@@ -38,7 +38,7 @@
|
||||
},
|
||||
"args": [
|
||||
"-A",
|
||||
"application.app.celery",
|
||||
"docsgpt.app.celery",
|
||||
"worker",
|
||||
"-l",
|
||||
"INFO",
|
||||
|
||||
@@ -28,22 +28,32 @@ Use these commands once the dev prerequisites above are satisfied.
|
||||
|
||||
```bash
|
||||
source .venv/bin/activate # macOS/Linux
|
||||
uv pip install -r application/requirements.txt # or: pip install -r application/requirements.txt
|
||||
# Optional docling parser engine (OCR / read_document structured output):
|
||||
# uv pip install -r application/requirements-docling.txt
|
||||
uv pip install -r docsgpt/requirements.txt # or: pip install -r docsgpt/requirements.txt
|
||||
# Optional extras (not installed by default; each file = core + the extra):
|
||||
# uv pip install -r docsgpt/requirements-docling.txt # docling parser engine (OCR backend, structured output)
|
||||
# uv pip install -r docsgpt/requirements-milvus.txt # VECTOR_STORE=milvus
|
||||
# With uv alone: `uv sync --extra docling` (pyproject.toml + uv.lock are the source of truth).
|
||||
# `uv pip install -r docsgpt/requirements-docling.txt` needs UV_INDEX_STRATEGY=unsafe-best-match
|
||||
# (the file adds the PyTorch CPU index; prefer `uv sync --extra docling`).
|
||||
```
|
||||
|
||||
Dependencies are declared in `pyproject.toml` and locked in `uv.lock`; the
|
||||
`docsgpt/requirements*.txt` files are exported from the lock. To add or
|
||||
bump a package: edit `pyproject.toml`, run `uv lock`, then
|
||||
`bash scripts/export_requirements.sh` (CI fails if the exports are stale).
|
||||
Never edit the requirements files by hand.
|
||||
|
||||
Run the API. For local dev, prefer the ASGI entrypoint under uvicorn — it
|
||||
serves the **whole** app, matches production, and hot-reloads:
|
||||
|
||||
```bash
|
||||
uvicorn application.asgi:asgi_app --host 0.0.0.0 --port 7091 --reload
|
||||
uvicorn docsgpt.asgi:asgi_app --host 0.0.0.0 --port 7091 --reload
|
||||
```
|
||||
|
||||
`flask --app application/app.py run --host=0.0.0.0 --port=7091` is a faster
|
||||
`flask --app docsgpt/app.py run --host=0.0.0.0 --port=7091` is a faster
|
||||
inner loop (quick startup, the Werkzeug interactive debugger), but it serves
|
||||
**only** the WSGI Flask app and omits the routes mounted on the ASGI shell
|
||||
in `application/asgi.py`:
|
||||
in `docsgpt/asgi.py`:
|
||||
|
||||
- the `/mcp` FastMCP endpoint, and
|
||||
- the native-async SSE reconnect reader `GET /api/messages/<id>/events`.
|
||||
@@ -53,13 +63,13 @@ Flask route), but a stream interrupted by a disconnect won't auto-resume on
|
||||
reconnect. Use `flask run` only when you don't need those routes.
|
||||
|
||||
Production uses `gunicorn -k uvicorn_worker.UvicornWorker` against the same
|
||||
`application.asgi:asgi_app` target; see `application/Dockerfile` for the
|
||||
`docsgpt.asgi:asgi_app` target; see `docsgpt/Dockerfile` for the
|
||||
full flag set.
|
||||
|
||||
Run the Celery worker in a separate terminal:
|
||||
|
||||
```bash
|
||||
celery -A application.app.celery worker -l INFO
|
||||
celery -A docsgpt.app.celery worker -l INFO
|
||||
```
|
||||
|
||||
**The worker is required for retrieval, not optional.** `EMBEDDINGS_DELEGATE_TO_WORKER`
|
||||
@@ -73,7 +83,7 @@ loading a model of its own — which keeps the API process around 285 MB instead
|
||||
On macOS, prefer the solo pool for Celery:
|
||||
|
||||
```bash
|
||||
python -m celery -A application.app.celery worker -l INFO --pool=solo
|
||||
python -m celery -A docsgpt.app.celery worker -l INFO --pool=solo
|
||||
```
|
||||
|
||||
Note that `--pool=solo` costs roughly 350 ms per query embed against ~55 ms on the
|
||||
@@ -147,7 +157,7 @@ vale .
|
||||
|
||||
## Repository map
|
||||
|
||||
- `application/`: Flask backend, API routes, agent logic, retrieval, parsing, security, storage, Celery worker, and WSGI entrypoints.
|
||||
- `docsgpt/`: Flask backend, API routes, agent logic, retrieval, parsing, security, storage, Celery worker, and WSGI entrypoints.
|
||||
- `tests/`: backend unit/integration tests and test-only Python dependencies.
|
||||
- `frontend/`: Vite + React + TypeScript application.
|
||||
- `frontend/src/`: main UI code, including `components`, `conversation`, `hooks`, `locale`, `settings`, `upload`, and Redux store wiring in `store.ts`.
|
||||
@@ -167,12 +177,12 @@ vale .
|
||||
|
||||
### Backend Abstractions
|
||||
|
||||
- LLM providers implement a common interface in `application/llm/` (add new providers by extending the base class).
|
||||
- Vector stores are abstracted in `application/vectorstore/`.
|
||||
- Parsers live in `application/parser/` and handle different document formats in the ingestion stage.
|
||||
- Agents and tools are in `application/agents/` and `application/agents/tools/`.
|
||||
- Celery setup/config lives in `application/celery_init.py` and `application/celeryconfig.py`.
|
||||
- Settings and env vars are managed via Pydantic in `application/core/settings.py`.
|
||||
- LLM providers implement a common interface in `docsgpt/llm/` (add new providers by extending the base class).
|
||||
- Vector stores are abstracted in `docsgpt/vectorstore/`.
|
||||
- Parsers live in `docsgpt/parser/` and handle different document formats in the ingestion stage.
|
||||
- Agents and tools are in `docsgpt/agents/` and `docsgpt/agents/tools/`.
|
||||
- Celery setup/config lives in `docsgpt/celery_init.py` and `docsgpt/celeryconfig.py`.
|
||||
- Settings and env vars are managed via Pydantic in `docsgpt/core/settings.py`.
|
||||
|
||||
### Frontend
|
||||
|
||||
|
||||
+1
-1
@@ -49,7 +49,7 @@ Tech Stack Overview:
|
||||
|
||||
### 🖥 Backend Contributions (🐍 Python)
|
||||
|
||||
- Review our issues and contribute to [`/application`](https://github.com/arc53/DocsGPT/tree/main/application)
|
||||
- Review our issues and contribute to [`/docsgpt`](https://github.com/arc53/DocsGPT/tree/main/docsgpt)
|
||||
- All new code should be covered with unit tests ([pytest](https://github.com/pytest-dev/pytest)). Please find tests under [`/tests`](https://github.com/arc53/DocsGPT/tree/main/tests) folder.
|
||||
- Before submitting your Pull Request, ensure it can be queried after ingesting some test data.
|
||||
- **Coding Style:** We adhere to the [PEP 8](https://www.python.org/dev/peps/pep-0008/) style guide for Python code. We use `ruff` as our linter and code formatter. Please ensure your code is formatted correctly and passes `ruff` checks before submitting.
|
||||
|
||||
@@ -132,7 +132,7 @@ Please refer to the [CONTRIBUTING.md](CONTRIBUTING.md) file for information abou
|
||||
|
||||
## Project Structure
|
||||
|
||||
- **Application** - Backend Flask application.
|
||||
- **docsgpt** - Backend Flask application (the `docsgpt` Python package).
|
||||
|
||||
- **Extensions** - Integrations and widgets (e.g., Chatwoot, React widget).
|
||||
|
||||
|
||||
@@ -1,148 +0,0 @@
|
||||
# Builder Stage
|
||||
FROM ubuntu:24.04 as builder
|
||||
|
||||
ENV DEBIAN_FRONTEND=noninteractive
|
||||
|
||||
RUN apt-get update && \
|
||||
apt-get install -y software-properties-common && \
|
||||
add-apt-repository ppa:deadsnakes/ppa && \
|
||||
apt-get update && \
|
||||
apt-get install -y --no-install-recommends gcc g++ wget unzip libc6-dev python3.12 python3.12-venv python3.12-dev && \
|
||||
rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# Verify Python installation and setup symlink
|
||||
RUN if [ -f /usr/bin/python3.12 ]; then \
|
||||
ln -s /usr/bin/python3.12 /usr/bin/python; \
|
||||
else \
|
||||
echo "Python 3.12 not found"; exit 1; \
|
||||
fi
|
||||
|
||||
# Install Rust
|
||||
RUN wget -q -O - https://sh.rustup.rs | sh -s -- -y
|
||||
|
||||
# Clean up to reduce container size
|
||||
RUN apt-get remove --purge -y wget unzip && apt-get autoremove -y && rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# Copy requirements manifests
|
||||
COPY requirements.txt requirements-docling.txt ./
|
||||
|
||||
# Setup Python virtual environment
|
||||
RUN python3.12 -m venv /venv
|
||||
|
||||
# Activate virtual environment and install Python packages
|
||||
ENV PATH="/venv/bin:$PATH"
|
||||
|
||||
# Install Python packages
|
||||
RUN pip install --no-cache-dir --upgrade pip && \
|
||||
pip install --no-cache-dir tiktoken && \
|
||||
pip install --no-cache-dir -r requirements.txt
|
||||
|
||||
# Optional docling parser engine (DOC_PARSER_ENGINE=docling, the docling OCR
|
||||
# backend, read_document's structured output) — OFF by default: it pulls the
|
||||
# layout/OCR model stack and adds gigabytes to the image. anydoc (in
|
||||
# requirements.txt) is the default parser and needs none of it, and OCR runs
|
||||
# natively on tesseract (below, also opt-in) without it.
|
||||
ARG INSTALL_DOCLING=false
|
||||
RUN if [ "$INSTALL_DOCLING" = "true" ]; then \
|
||||
pip install --no-cache-dir -r requirements-docling.txt; \
|
||||
fi
|
||||
|
||||
# Final Stage
|
||||
FROM ubuntu:24.04 as final
|
||||
|
||||
RUN apt-get update && \
|
||||
apt-get install -y software-properties-common && \
|
||||
add-apt-repository ppa:deadsnakes/ppa && \
|
||||
apt-get update && apt-get install -y --no-install-recommends \
|
||||
python3.12 \
|
||||
libgl1 \
|
||||
libglib2.0-0 \
|
||||
poppler-utils \
|
||||
&& \
|
||||
ln -s /usr/bin/python3.12 /usr/bin/python && \
|
||||
rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# Optional tesseract OCR engine (OCR_ENABLED=true with OCR_ENGINE=tesseract,
|
||||
# the default engine) — OFF by default like every other OCR dependency; OCR
|
||||
# itself is off unless configured. Opt in with --build-arg
|
||||
# INSTALL_TESSERACT=true (setup.sh writes it to .env when OCR is enabled);
|
||||
# ~35 MB of system packages. Extra language packs are a deployment concern
|
||||
# (apt: tesseract-ocr-<lang>, then list them in OCR_LANGS). A DeepSeek-OCR
|
||||
# endpoint (OCR_ENGINE=deepseek) needs none of this.
|
||||
ARG INSTALL_TESSERACT=false
|
||||
RUN if [ "$INSTALL_TESSERACT" = "true" ]; then \
|
||||
apt-get update && \
|
||||
apt-get install -y --no-install-recommends tesseract-ocr tesseract-ocr-eng && \
|
||||
rm -rf /var/lib/apt/lists/*; \
|
||||
fi
|
||||
|
||||
# Set working directory
|
||||
WORKDIR /app
|
||||
|
||||
# Create a non-root user: `appuser` (Feel free to choose a name)
|
||||
RUN groupadd -r appuser && \
|
||||
useradd -r -g appuser -d /app -s /sbin/nologin -c "Docker image user" appuser
|
||||
|
||||
# Copy the virtual environment and model from the builder stage
|
||||
COPY --from=builder /venv /venv
|
||||
|
||||
# Pre-fetch the embedding models into FastEmbed's cache so a fresh container
|
||||
# does not download on first ingest and an air-gapped install works at all.
|
||||
# Both defaults are baked: an upgraded deployment keeps using mpnet until it
|
||||
# runs the re-embed script, while a new one starts on granite.
|
||||
# The prefetch writes hub-layout snapshots (including tokenizer.json) here, so
|
||||
# HF_HUB_CACHE has to point at the same directory: chunking loads the tokenizer
|
||||
# through ``tokenizers``, which reads the hub cache and would otherwise fetch
|
||||
# over the network on first ingest -- and fall back to cl100k when offline.
|
||||
ENV EMBEDDINGS_CACHE_DIR=/app/models \
|
||||
HF_HUB_CACHE=/app/models
|
||||
|
||||
# Only the modules the prefetch imports are copied first. It reaches nothing
|
||||
# beyond model_registry, which is stdlib-only, so keeping the full source copy
|
||||
# below this layer stops an unrelated edit from re-downloading ~780 MB of model
|
||||
# artifacts on every build.
|
||||
COPY __init__.py /app/application/__init__.py
|
||||
COPY scripts/__init__.py scripts/prefetch_models.py /app/application/scripts/
|
||||
COPY vectorstore/__init__.py vectorstore/model_registry.py /app/application/vectorstore/
|
||||
ARG EMBEDDINGS_PREFETCH=""
|
||||
RUN PYTHONPATH=/app /venv/bin/python -m application.scripts.prefetch_models ${EMBEDDINGS_PREFETCH}
|
||||
|
||||
# Copy your application code
|
||||
COPY . /app/application
|
||||
|
||||
# Change the ownership of the /app directory to the appuser
|
||||
|
||||
RUN mkdir -p /app/application/inputs/local
|
||||
RUN chown -R appuser:appuser /app
|
||||
|
||||
# Set environment variables
|
||||
ENV FLASK_APP=app.py \
|
||||
FLASK_DEBUG=true \
|
||||
PATH="/venv/bin:$PATH"
|
||||
|
||||
ENV MALLOC_ARENA_MAX=2 \
|
||||
OMP_NUM_THREADS=4 \
|
||||
MKL_NUM_THREADS=4 \
|
||||
OPENBLAS_NUM_THREADS=4
|
||||
|
||||
# Expose the port the app runs on
|
||||
EXPOSE 7091
|
||||
|
||||
# Switch to non-root user
|
||||
USER appuser
|
||||
|
||||
# BoundedDrainUvicornWorker makes max_requests recycles safe with held-open SSE
|
||||
# connections (see application/gunicorn_worker.py); with recycles now safe,
|
||||
# --max-requests is raised (kept for memory hygiene) to cut churn.
|
||||
CMD ["gunicorn", \
|
||||
"-w", "1", \
|
||||
"-k", "application.gunicorn_worker.BoundedDrainUvicornWorker", \
|
||||
"--bind", "0.0.0.0:7091", \
|
||||
"--timeout", "180", \
|
||||
"--graceful-timeout", "120", \
|
||||
"--keep-alive", "5", \
|
||||
"--worker-tmp-dir", "/dev/shm", \
|
||||
"--max-requests", "5000", \
|
||||
"--max-requests-jitter", "500", \
|
||||
"--config", "application/gunicorn_conf.py", \
|
||||
"application.asgi:asgi_app"]
|
||||
@@ -0,0 +1,77 @@
|
||||
"""``application`` is now ``docsgpt``; this alias keeps the old name importable for one release.
|
||||
|
||||
Every ``import application.x.y`` resolves to the already-imported ``docsgpt.x.y``
|
||||
module object, so there is exactly one settings object, one Celery app and one
|
||||
Flask app however a process refers to them. Entry points such as
|
||||
``celery -A application.app.celery``, ``uvicorn application.asgi:asgi_app`` and
|
||||
``python -m application.scripts.<name>`` keep working; update them to
|
||||
``docsgpt.…`` before the alias is removed.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib
|
||||
import importlib.abc
|
||||
import importlib.util
|
||||
import sys
|
||||
import warnings
|
||||
|
||||
_OLD = __name__
|
||||
_NEW = "docsgpt"
|
||||
|
||||
|
||||
class _AliasLoader(importlib.abc.Loader):
|
||||
"""Hand back the ``docsgpt`` module object; delegate code access to its real loader."""
|
||||
|
||||
def __init__(self, target: str, target_spec) -> None:
|
||||
self._target = target
|
||||
self._target_spec = target_spec
|
||||
self._original_spec = None
|
||||
|
||||
def create_module(self, spec):
|
||||
module = importlib.import_module(self._target)
|
||||
self._original_spec = module.__spec__
|
||||
return module
|
||||
|
||||
def exec_module(self, module) -> None:
|
||||
# The import machinery stamps the alias spec on the shared module
|
||||
# object; put the real one back so importlib.reload and __spec__-based
|
||||
# lookups keep addressing the module by its docsgpt name.
|
||||
if self._original_spec is not None:
|
||||
module.__spec__ = self._original_spec
|
||||
|
||||
# runpy (``python -m application.x``) reads the code through the loader.
|
||||
def get_code(self, fullname):
|
||||
return self._target_spec.loader.get_code(self._target)
|
||||
|
||||
def get_source(self, fullname):
|
||||
return self._target_spec.loader.get_source(self._target)
|
||||
|
||||
def get_filename(self, fullname):
|
||||
return self._target_spec.loader.get_filename(self._target)
|
||||
|
||||
def is_package(self, fullname):
|
||||
return self._target_spec.submodule_search_locations is not None
|
||||
|
||||
|
||||
class _AliasFinder(importlib.abc.MetaPathFinder):
|
||||
"""Resolve ``application.<path>`` to ``docsgpt.<path>``."""
|
||||
|
||||
def find_spec(self, name, path=None, target=None):
|
||||
if name != _OLD and not name.startswith(_OLD + "."):
|
||||
return None
|
||||
new_name = _NEW + name[len(_OLD):]
|
||||
target_spec = importlib.util.find_spec(new_name)
|
||||
if target_spec is None:
|
||||
return None
|
||||
return importlib.util.spec_from_loader(name, _AliasLoader(new_name, target_spec))
|
||||
|
||||
|
||||
warnings.warn(
|
||||
"The 'application' package was renamed to 'docsgpt'. Update imports and entry points "
|
||||
"(celery -A docsgpt.app.celery, uvicorn docsgpt.asgi:asgi_app); this alias will be removed.",
|
||||
FutureWarning,
|
||||
stacklevel=2,
|
||||
)
|
||||
sys.meta_path.insert(0, _AliasFinder())
|
||||
sys.modules[_OLD] = importlib.import_module(_NEW)
|
||||
@@ -1,3 +0,0 @@
|
||||
from application.api.v1.routes import v1_bp
|
||||
|
||||
__all__ = ["v1_bp"]
|
||||
@@ -1,15 +0,0 @@
|
||||
# Optional docling parser engine.
|
||||
#
|
||||
# Needed only for: DOC_PARSER_ENGINE=docling, the docling OCR backend
|
||||
# (OCR_BACKEND=docling: layout-model hybrid OCR, ocrmac/rapidocr engines),
|
||||
# .adoc/.vtt/.xml attachment parsing, and the read_document tool's `structured`
|
||||
# output. The default anydoc engine needs none of this, and OCR itself does
|
||||
# not either: OCR_ENABLED=true with the tesseract binary (or a DeepSeek-OCR
|
||||
# endpoint) runs through application/parser/file/ocr_parser.py.
|
||||
#
|
||||
# Install ON TOP of requirements.txt, which pins torch, transformers and
|
||||
# onnxruntime (rapidocr runs on the same onnxruntime fastembed uses):
|
||||
# pip install -r requirements.txt -r requirements-docling.txt
|
||||
# Docker: build with --build-arg INSTALL_DOCLING=true
|
||||
docling==2.119.0
|
||||
rapidocr==3.9.2
|
||||
@@ -1,125 +0,0 @@
|
||||
a2wsgi==1.10.10
|
||||
alembic>=1.13,<2
|
||||
anthropic==0.121.0
|
||||
asgiref>=3.11.1
|
||||
boto3==1.43.67
|
||||
beautifulsoup4==4.15.0
|
||||
cel-python==0.5.0
|
||||
celery==5.6.3
|
||||
celery-redbeat==2.4.2
|
||||
croniter==6.2.4
|
||||
cryptography==50.0.0
|
||||
dataclasses-json==0.6.7
|
||||
daytona==0.205.1
|
||||
defusedxml==0.7.1
|
||||
# Default document converter (DOC_PARSER_ENGINE=anydoc). The docling engine
|
||||
# (OCR, layout/table models, read_document's structured output) is an
|
||||
# optional extra: pip install -r application/requirements-docling.txt
|
||||
firecrawl-anydoc==0.2.3
|
||||
docx2txt==0.9
|
||||
ddgs>=8.0.0
|
||||
fast-ebook
|
||||
elevenlabs==2.62.0
|
||||
Flask==3.1.3
|
||||
fastembed==0.8.0
|
||||
# fastembed's runtime: local embeddings execute on it, so it stays pinned in
|
||||
# core even though the docling extra's rapidocr runs on the same package.
|
||||
onnxruntime==1.28.0
|
||||
faiss-cpu==1.15.0
|
||||
fastmcp==3.4.6
|
||||
flask-restx==1.3.2
|
||||
google-genai==2.17.0
|
||||
google-api-python-client==2.198.0
|
||||
google-auth-httplib2==0.4.1
|
||||
google-auth-oauthlib==1.4.0
|
||||
gTTS==2.5.4
|
||||
gunicorn==26.0.0
|
||||
jinja2==3.1.6
|
||||
jiter==0.16.0
|
||||
jmespath==1.1.0
|
||||
joblib==1.5.3
|
||||
jsonpatch==1.33
|
||||
jsonpointer==3.1.1
|
||||
kombu==5.6.2
|
||||
lazy-object-proxy==1.12.0
|
||||
lxml==6.1.1
|
||||
markupsafe==3.0.3
|
||||
marshmallow>=3.18.0,<4.0.0
|
||||
mpmath==1.3.0
|
||||
multidict==6.7.1
|
||||
msal==1.37.0
|
||||
mypy-extensions==1.1.0
|
||||
networkx==3.6.1
|
||||
numpy==2.5.1
|
||||
openai==2.53.0
|
||||
openapi3-parser==1.1.22
|
||||
opentelemetry-distro>=0.50b0,<1
|
||||
opentelemetry-exporter-otlp>=1.29.0,<2
|
||||
opentelemetry-instrumentation-celery>=0.50b0,<1
|
||||
opentelemetry-instrumentation-flask>=0.50b0,<1
|
||||
opentelemetry-instrumentation-logging>=0.50b0,<1
|
||||
opentelemetry-instrumentation-psycopg>=0.50b0,<1
|
||||
opentelemetry-instrumentation-redis>=0.50b0,<1
|
||||
opentelemetry-instrumentation-requests>=0.50b0,<1
|
||||
opentelemetry-instrumentation-sqlalchemy>=0.50b0,<1
|
||||
opentelemetry-instrumentation-starlette>=0.50b0,<1
|
||||
orjson==3.11.9
|
||||
packaging==26.3
|
||||
pandas==3.0.5
|
||||
openpyxl==3.1.5
|
||||
pathable==0.6.0
|
||||
pdf2image>=1.17.0
|
||||
pgvector>=0.5,<1
|
||||
pillow==12.3.0
|
||||
portalocker>=2.7.0,<4.0.0
|
||||
prompt-toolkit==3.0.53
|
||||
protobuf==7.35.1
|
||||
psycopg[binary,pool]>=3.1,<4
|
||||
py==1.11.0
|
||||
pydantic
|
||||
pydantic-core
|
||||
pydantic-settings
|
||||
praw==8.0.2
|
||||
pymilvus==3.0.1
|
||||
milvus-lite==3.2.0; sys_platform != "win32"
|
||||
pypdf==6.15.0
|
||||
pypdfium2==5.12.1
|
||||
qdrant-client==1.19.0
|
||||
python-dateutil==2.9.0.post0
|
||||
python-dotenv
|
||||
python-jose==3.5.0
|
||||
python-pptx==1.0.2
|
||||
PyYAML
|
||||
redis==7.4.0
|
||||
referencing>=0.28.0,<0.38.0
|
||||
regex==2026.7.19
|
||||
requests==2.34.2
|
||||
retry==0.9.2
|
||||
sqlalchemy>=2.0,<3
|
||||
starlette>=1.0,<2
|
||||
tiktoken==0.13.0
|
||||
tokenizers==0.22.2
|
||||
# torch and transformers serve docling's model stack alone -- embeddings moved
|
||||
# to fastembed/onnxruntime. Still in core rather than requirements-docling.txt,
|
||||
# so installing the optional extra never re-resolves the shared pins; both can
|
||||
# move out with docling.
|
||||
torch==2.11.0
|
||||
tqdm==4.67.3
|
||||
# Capped <5.9.0 to match docling-core's own darwin pin: 5.9+ breaks docling's
|
||||
# PDF layout model on Apple Silicon ("Cannot convert a MPS Tensor to float64").
|
||||
transformers==5.8.1
|
||||
typing-extensions==4.16.0
|
||||
typing-inspect==0.9.0
|
||||
tzdata==2026.3
|
||||
urllib3==2.7.0
|
||||
uvicorn[standard]>=0.30,<1
|
||||
uvicorn-worker>=0.4,<1
|
||||
vine==5.1.0
|
||||
wcwidth==0.8.2
|
||||
werkzeug>=3.1.0
|
||||
yarl==1.24.5
|
||||
markdownify==1.2.3
|
||||
tldextract==5.3.2
|
||||
# google-genai caps websockets <17.0; 17.x will not resolve.
|
||||
websockets==16.0
|
||||
websocket-client==1.9.0
|
||||
@@ -1,9 +1,24 @@
|
||||
services:
|
||||
frontend:
|
||||
build: ../frontend
|
||||
build:
|
||||
context: ../frontend
|
||||
target: dev
|
||||
environment:
|
||||
# Every VITE_* the app reads. A bare name is passed through only when it is
|
||||
# set in the shell or the --env-file, so an unset one does not reach the
|
||||
# container as an empty string and override the image's own default.
|
||||
- VITE_API_HOST=http://localhost:7091
|
||||
- VITE_API_STREAMING=$VITE_API_STREAMING
|
||||
- VITE_API_STREAMING=${VITE_API_STREAMING:-true}
|
||||
- VITE_BASE_URL
|
||||
- VITE_GOOGLE_CLIENT_ID
|
||||
- VITE_GOOGLE_PICKER_API_KEY
|
||||
- VITE_SHARE_POINT_CLIENT_ID
|
||||
- VITE_CONFLUENCE_CLIENT_ID
|
||||
- VITE_NOTIFICATION_TEXT
|
||||
- VITE_NOTIFICATION_LINK
|
||||
- VITE_ENABLE_VOICE_INPUT
|
||||
- VITE_DISABLE_SOURCE_FE
|
||||
- VITE_USE_V
|
||||
ports:
|
||||
- "5173:5173"
|
||||
depends_on:
|
||||
@@ -11,8 +26,10 @@ services:
|
||||
|
||||
backend:
|
||||
build:
|
||||
context: ../application
|
||||
context: ..
|
||||
dockerfile: docsgpt/Dockerfile
|
||||
args:
|
||||
EXTRAS: ${EXTRAS:-}
|
||||
INSTALL_DOCLING: ${INSTALL_DOCLING:-false}
|
||||
# Off by default; deployments running OCR_ENABLED=true with tesseract
|
||||
# must set INSTALL_TESSERACT=true before rebuilding (see docker-compose.yaml).
|
||||
@@ -27,9 +44,14 @@ services:
|
||||
ports:
|
||||
- "7091:7091"
|
||||
volumes:
|
||||
- ../application/indexes:/app/application/indexes
|
||||
- ../application/inputs:/app/application/inputs
|
||||
- ../application/vectors:/app/application/vectors
|
||||
# Host data lives under application/ (the old package directory) for this
|
||||
# release, like the other compose files; it moves with the packaging work,
|
||||
# together with an upgrade note. The container side is /app: the process
|
||||
# runs there and resolves indexes/inputs/vectors relative to it (earlier
|
||||
# versions of this file mounted /app/application/..., which nothing wrote to).
|
||||
- ../application/indexes:/app/indexes
|
||||
- ../application/inputs:/app/inputs
|
||||
- ../application/vectors:/app/vectors
|
||||
depends_on:
|
||||
redis:
|
||||
condition: service_started
|
||||
@@ -38,14 +60,16 @@ services:
|
||||
|
||||
worker:
|
||||
build:
|
||||
context: ../application
|
||||
context: ..
|
||||
dockerfile: docsgpt/Dockerfile
|
||||
args:
|
||||
EXTRAS: ${EXTRAS:-}
|
||||
INSTALL_DOCLING: ${INSTALL_DOCLING:-false}
|
||||
# Off by default; deployments running OCR_ENABLED=true with tesseract
|
||||
# must set INSTALL_TESSERACT=true before rebuilding (see docker-compose.yaml).
|
||||
INSTALL_TESSERACT: ${INSTALL_TESSERACT:-false}
|
||||
# `parsing` queue carries read_document/parse_document; required for its await to resolve.
|
||||
command: celery -A application.app.celery worker -l INFO -Q docsgpt,parsing,embeddings
|
||||
command: celery -A docsgpt.app.celery worker -l INFO -Q docsgpt,parsing,embeddings
|
||||
env_file:
|
||||
- ../.env
|
||||
environment:
|
||||
|
||||
@@ -1,12 +1,30 @@
|
||||
# Pre-built images from Docker Hub (mirrored at ghcr.io/arc53).
|
||||
# DOCSGPT_IMAGE_TAG develop (default, follows main) or a release, e.g. 0.20.0
|
||||
# DOCSGPT_IMAGE_VARIANT empty (default, slim) or -docling: docling parser engine,
|
||||
# its models, and tesseract baked in (OCR-ready)
|
||||
# Set them in ../.env or the shell. deployment/docker-compose-standalone.yaml is
|
||||
# the same stack without a git checkout.
|
||||
name: docsgpt-oss
|
||||
services:
|
||||
|
||||
frontend:
|
||||
image: arc53/docsgpt-fe:develop
|
||||
image: arc53/docsgpt-fe:${DOCSGPT_IMAGE_TAG:-develop}
|
||||
environment:
|
||||
# Every VITE_* the app reads. A bare name is passed through only when it is
|
||||
# set in the shell or the --env-file, so an unset one does not reach the
|
||||
# container as an empty string and override the image's own default.
|
||||
- VITE_API_HOST=http://localhost:7091
|
||||
- VITE_API_STREAMING=${VITE_API_STREAMING:-true}
|
||||
- VITE_GOOGLE_CLIENT_ID=${VITE_GOOGLE_CLIENT_ID:-}
|
||||
- VITE_BASE_URL
|
||||
- VITE_GOOGLE_CLIENT_ID
|
||||
- VITE_GOOGLE_PICKER_API_KEY
|
||||
- VITE_SHARE_POINT_CLIENT_ID
|
||||
- VITE_CONFLUENCE_CLIENT_ID
|
||||
- VITE_NOTIFICATION_TEXT
|
||||
- VITE_NOTIFICATION_LINK
|
||||
- VITE_ENABLE_VOICE_INPUT
|
||||
- VITE_DISABLE_SOURCE_FE
|
||||
- VITE_USE_V
|
||||
ports:
|
||||
- "5173:5173"
|
||||
depends_on:
|
||||
@@ -15,7 +33,7 @@ services:
|
||||
|
||||
backend:
|
||||
user: root
|
||||
image: arc53/docsgpt:develop
|
||||
image: arc53/docsgpt:${DOCSGPT_IMAGE_TAG:-develop}${DOCSGPT_IMAGE_VARIANT:-}
|
||||
env_file:
|
||||
- ../.env
|
||||
environment:
|
||||
@@ -26,6 +44,9 @@ services:
|
||||
ports:
|
||||
- "7091:7091"
|
||||
volumes:
|
||||
# Host data stays under application/ (the old package directory) for this
|
||||
# release so an upgraded checkout keeps its indexes; it moves with the
|
||||
# packaging work, together with an upgrade note.
|
||||
- ../application/indexes:/app/indexes
|
||||
- ../application/inputs:/app/inputs
|
||||
- ../application/vectors:/app/vectors
|
||||
@@ -38,9 +59,9 @@ services:
|
||||
|
||||
worker:
|
||||
user: root
|
||||
image: arc53/docsgpt:develop
|
||||
image: arc53/docsgpt:${DOCSGPT_IMAGE_TAG:-develop}${DOCSGPT_IMAGE_VARIANT:-}
|
||||
# `parsing` queue carries read_document/parse_document; required for its await to resolve.
|
||||
command: celery -A application.app.celery worker -l INFO -B -Q docsgpt,parsing,embeddings
|
||||
command: celery -A docsgpt.app.celery worker -l INFO -B -Q docsgpt,parsing,embeddings
|
||||
env_file:
|
||||
- ../.env
|
||||
environment:
|
||||
@@ -50,6 +71,9 @@ services:
|
||||
- CACHE_REDIS_URL=redis://redis:6379/2
|
||||
- POSTGRES_URI=postgresql://docsgpt:docsgpt@postgres:5432/docsgpt
|
||||
volumes:
|
||||
# Host data stays under application/ (the old package directory) for this
|
||||
# release so an upgraded checkout keeps its indexes; it moves with the
|
||||
# packaging work, together with an upgrade note.
|
||||
- ../application/indexes:/app/indexes
|
||||
- ../application/inputs:/app/inputs
|
||||
- ../application/vectors:/app/vectors
|
||||
|
||||
@@ -0,0 +1,130 @@
|
||||
# DocsGPT from pre-built images, with no git checkout.
|
||||
#
|
||||
# curl -fsSLO https://raw.githubusercontent.com/arc53/DocsGPT/main/deployment/docker-compose-standalone.yaml
|
||||
# printf 'LLM_PROVIDER=docsgpt\nVITE_API_STREAMING=true\nINTERNAL_KEY=%s\n' "$(openssl rand -hex 16)" > .env
|
||||
# docker compose -f docker-compose-standalone.yaml up -d
|
||||
#
|
||||
# INTERNAL_KEY is the shared secret the worker uses to hand finished indexes to
|
||||
# the API; without it every ingest fails with a 401 (setup.sh generates one).
|
||||
# open http://localhost:5173
|
||||
#
|
||||
# Every release also attaches this file as an asset. Settings come from .env
|
||||
# next to this file (any DocsGPT setting; the compose-internal service URLs
|
||||
# below take precedence). Data lives in named volumes, so `docker compose
|
||||
# down` keeps it and `docker compose down -v` removes it.
|
||||
#
|
||||
# DOCSGPT_IMAGE_TAG release to run, e.g. 0.20.0 (default: latest release);
|
||||
# develop follows the main branch
|
||||
# DOCSGPT_IMAGE_VARIANT empty (slim, default) or -docling: docling parser
|
||||
# engine, its models, and tesseract baked in (OCR-ready)
|
||||
# EMBEDDINGS_NAME defaults to granite here (this stack always starts on
|
||||
# fresh volumes, so there is no older index to keep
|
||||
# compatible); the code default stays mpnet for upgrades.
|
||||
name: docsgpt
|
||||
|
||||
services:
|
||||
frontend:
|
||||
image: arc53/docsgpt-fe:${DOCSGPT_IMAGE_TAG:-latest}
|
||||
env_file:
|
||||
- path: .env
|
||||
required: false
|
||||
environment:
|
||||
# Every VITE_* the app reads. A bare name is passed through only when it is
|
||||
# set in the shell or the --env-file, so an unset one does not reach the
|
||||
# container as an empty string and override the image's own default.
|
||||
- VITE_API_HOST=${VITE_API_HOST:-http://localhost:7091}
|
||||
- VITE_API_STREAMING=${VITE_API_STREAMING:-true}
|
||||
- VITE_BASE_URL
|
||||
- VITE_GOOGLE_CLIENT_ID
|
||||
- VITE_GOOGLE_PICKER_API_KEY
|
||||
- VITE_SHARE_POINT_CLIENT_ID
|
||||
- VITE_CONFLUENCE_CLIENT_ID
|
||||
- VITE_NOTIFICATION_TEXT
|
||||
- VITE_NOTIFICATION_LINK
|
||||
- VITE_ENABLE_VOICE_INPUT
|
||||
- VITE_DISABLE_SOURCE_FE
|
||||
- VITE_USE_V
|
||||
ports:
|
||||
- "5173:5173"
|
||||
depends_on:
|
||||
- backend
|
||||
|
||||
backend:
|
||||
image: arc53/docsgpt:${DOCSGPT_IMAGE_TAG:-latest}${DOCSGPT_IMAGE_VARIANT:-}
|
||||
# Same as docker-compose-hub.yaml: the data volumes are written by root so
|
||||
# any image tag works, including releases that predate the appuser-owned
|
||||
# /app/inputs, /app/indexes and /app/vectors directories.
|
||||
user: root
|
||||
env_file:
|
||||
- path: .env
|
||||
required: false
|
||||
environment:
|
||||
- CELERY_BROKER_URL=redis://redis:6379/0
|
||||
- CELERY_RESULT_BACKEND=redis://redis:6379/1
|
||||
- CACHE_REDIS_URL=redis://redis:6379/2
|
||||
- POSTGRES_URI=postgresql://docsgpt:docsgpt@postgres:5432/docsgpt
|
||||
- EMBEDDINGS_NAME=${EMBEDDINGS_NAME:-ibm-granite/granite-embedding-311m-multilingual-r2}
|
||||
ports:
|
||||
- "7091:7091"
|
||||
volumes:
|
||||
- indexes:/app/indexes
|
||||
- inputs:/app/inputs
|
||||
- vectors:/app/vectors
|
||||
depends_on:
|
||||
redis:
|
||||
condition: service_started
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
restart: unless-stopped
|
||||
|
||||
worker:
|
||||
image: arc53/docsgpt:${DOCSGPT_IMAGE_TAG:-latest}${DOCSGPT_IMAGE_VARIANT:-}
|
||||
user: root
|
||||
# Consumes the default queue plus `parsing` (read_document) and `embeddings`
|
||||
# (query embedding); without the latter every search times out.
|
||||
command: celery -A docsgpt.app.celery worker -l INFO -B -Q docsgpt,parsing,embeddings
|
||||
env_file:
|
||||
- path: .env
|
||||
required: false
|
||||
environment:
|
||||
- CELERY_BROKER_URL=redis://redis:6379/0
|
||||
- CELERY_RESULT_BACKEND=redis://redis:6379/1
|
||||
- CACHE_REDIS_URL=redis://redis:6379/2
|
||||
- POSTGRES_URI=postgresql://docsgpt:docsgpt@postgres:5432/docsgpt
|
||||
- API_URL=http://backend:7091
|
||||
- EMBEDDINGS_NAME=${EMBEDDINGS_NAME:-ibm-granite/granite-embedding-311m-multilingual-r2}
|
||||
volumes:
|
||||
- indexes:/app/indexes
|
||||
- inputs:/app/inputs
|
||||
- vectors:/app/vectors
|
||||
depends_on:
|
||||
redis:
|
||||
condition: service_started
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
restart: unless-stopped
|
||||
|
||||
redis:
|
||||
image: redis:6-alpine
|
||||
restart: unless-stopped
|
||||
|
||||
postgres:
|
||||
image: postgres:16-alpine
|
||||
environment:
|
||||
- POSTGRES_USER=docsgpt
|
||||
- POSTGRES_PASSWORD=docsgpt
|
||||
- POSTGRES_DB=docsgpt
|
||||
volumes:
|
||||
- postgres_data:/var/lib/postgresql/data
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "pg_isready -U docsgpt -d docsgpt"]
|
||||
interval: 5s
|
||||
timeout: 5s
|
||||
retries: 10
|
||||
restart: unless-stopped
|
||||
|
||||
volumes:
|
||||
indexes:
|
||||
inputs:
|
||||
vectors:
|
||||
postgres_data:
|
||||
@@ -1,13 +1,29 @@
|
||||
name: docsgpt-oss
|
||||
services:
|
||||
frontend:
|
||||
build: ../frontend
|
||||
build:
|
||||
context: ../frontend
|
||||
# Vite dev server with hot reload over the bind mount below. The default
|
||||
# target (what Docker Hub publishes) is a static build behind nginx.
|
||||
target: dev
|
||||
volumes:
|
||||
- ../frontend/src:/app/src
|
||||
environment:
|
||||
# Every VITE_* the app reads. A bare name is passed through only when it is
|
||||
# set in the shell or the --env-file, so an unset one does not reach the
|
||||
# container as an empty string and override the image's own default.
|
||||
- VITE_API_HOST=http://localhost:7091
|
||||
- VITE_API_STREAMING=$VITE_API_STREAMING
|
||||
- VITE_GOOGLE_CLIENT_ID=$VITE_GOOGLE_CLIENT_ID
|
||||
- VITE_API_STREAMING=${VITE_API_STREAMING:-true}
|
||||
- VITE_BASE_URL
|
||||
- VITE_GOOGLE_CLIENT_ID
|
||||
- VITE_GOOGLE_PICKER_API_KEY
|
||||
- VITE_SHARE_POINT_CLIENT_ID
|
||||
- VITE_CONFLUENCE_CLIENT_ID
|
||||
- VITE_NOTIFICATION_TEXT
|
||||
- VITE_NOTIFICATION_LINK
|
||||
- VITE_ENABLE_VOICE_INPUT
|
||||
- VITE_DISABLE_SOURCE_FE
|
||||
- VITE_USE_V
|
||||
ports:
|
||||
- "5173:5173"
|
||||
depends_on:
|
||||
@@ -16,10 +32,13 @@ services:
|
||||
backend:
|
||||
user: root
|
||||
build:
|
||||
context: ../application
|
||||
context: ..
|
||||
dockerfile: docsgpt/Dockerfile
|
||||
args:
|
||||
# Bake the optional docling engine (layout-model OCR backend, structured
|
||||
# output) into the image: set INSTALL_DOCLING=true in ../.env or the shell.
|
||||
# Optional extras to bake in (comma-separated): docling, milvus. The
|
||||
# docling extra brings the layout-model parser/OCR backend and its
|
||||
# models. INSTALL_DOCLING=true is the older spelling of EXTRAS=docling.
|
||||
EXTRAS: ${EXTRAS:-}
|
||||
INSTALL_DOCLING: ${INSTALL_DOCLING:-false}
|
||||
# Bake the tesseract binary behind OCR_ENABLED=true (~35 MB): set
|
||||
# INSTALL_TESSERACT=true in ../.env or the shell (setup.sh does this
|
||||
@@ -39,6 +58,9 @@ services:
|
||||
ports:
|
||||
- "7091:7091"
|
||||
volumes:
|
||||
# Host data stays under application/ (the old package directory) for this
|
||||
# release so an upgraded checkout keeps its indexes; it moves with the
|
||||
# packaging work, together with an upgrade note.
|
||||
- ../application/indexes:/app/indexes
|
||||
- ../application/inputs:/app/inputs
|
||||
- ../application/vectors:/app/vectors
|
||||
@@ -51,8 +73,10 @@ services:
|
||||
worker:
|
||||
user: root
|
||||
build:
|
||||
context: ../application
|
||||
context: ..
|
||||
dockerfile: docsgpt/Dockerfile
|
||||
args:
|
||||
EXTRAS: ${EXTRAS:-}
|
||||
INSTALL_DOCLING: ${INSTALL_DOCLING:-false}
|
||||
INSTALL_TESSERACT: ${INSTALL_TESSERACT:-false}
|
||||
# Consumes the default queue AND the dedicated `parsing` (read_document /
|
||||
@@ -61,7 +85,7 @@ services:
|
||||
# fails after EMBEDDINGS_DELEGATE_TIMEOUT, because EMBEDDINGS_DELEGATE_TO_WORKER
|
||||
# is on by default. For heavy/OCR parsing run a separate worker with `-Q parsing`;
|
||||
# to keep query latency off the ingest pool, another with `-Q embeddings`.
|
||||
command: celery -A application.app.celery worker -l INFO -B -Q docsgpt,parsing,embeddings
|
||||
command: celery -A docsgpt.app.celery worker -l INFO -B -Q docsgpt,parsing,embeddings
|
||||
env_file:
|
||||
- ../.env
|
||||
environment:
|
||||
@@ -72,6 +96,9 @@ services:
|
||||
- CACHE_REDIS_URL=redis://redis:6379/2
|
||||
- POSTGRES_URI=postgresql://docsgpt:docsgpt@postgres:5432/docsgpt
|
||||
volumes:
|
||||
# Host data stays under application/ (the old package directory) for this
|
||||
# release so an upgraded checkout keeps its indexes; it moves with the
|
||||
# packaging work, together with an upgrade note.
|
||||
- ../application/indexes:/app/indexes
|
||||
- ../application/inputs:/app/inputs
|
||||
- ../application/vectors:/app/vectors
|
||||
|
||||
@@ -42,7 +42,7 @@ spec:
|
||||
name: docsgpt-secrets
|
||||
env:
|
||||
- name: FLASK_APP
|
||||
value: "application/app.py"
|
||||
value: "docsgpt/app.py"
|
||||
- name: DEPLOYMENT_TYPE
|
||||
value: "cloud"
|
||||
- name: POSTGRES_URI
|
||||
@@ -87,7 +87,7 @@ spec:
|
||||
image: arc53/docsgpt
|
||||
# `parsing` queue carries read_document/parse_document; required for its await to resolve.
|
||||
# For heavy/OCR parsing, run a separate deployment with `-Q parsing` (and GPU env).
|
||||
command: ["celery", "-A", "application.app.celery", "worker", "-l", "INFO", "-n", "worker.%h", "-Q", "docsgpt,parsing,embeddings"]
|
||||
command: ["celery", "-A", "docsgpt.app.celery", "worker", "-l", "INFO", "-n", "worker.%h", "-Q", "docsgpt,parsing,embeddings"]
|
||||
resources:
|
||||
limits:
|
||||
memory: "4Gi"
|
||||
|
||||
@@ -35,7 +35,7 @@ spec:
|
||||
name: docsgpt-secrets
|
||||
env:
|
||||
- name: FLASK_APP
|
||||
value: "application/app.py"
|
||||
value: "docsgpt/app.py"
|
||||
resources:
|
||||
limits:
|
||||
memory: "1Gi"
|
||||
|
||||
@@ -190,7 +190,7 @@ Document reading no longer runs in this sandbox. The `read_document` tool and th
|
||||
workflow native-file extract branch enqueue a `parse_document` Celery task that
|
||||
parses the document **in the backend** (the `DOC_PARSER_ENGINE` parser — anydoc
|
||||
by default; Docling only when the optional
|
||||
`application/requirements-docling.txt` extra is installed) and awaits the
|
||||
`docsgpt/requirements-docling.txt` extra is installed) and awaits the
|
||||
result. The task is routed to a
|
||||
dedicated **`parsing` queue** (`settings.DOCUMENT_PARSE_QUEUE`, default
|
||||
`"parsing"`) so a parse enqueued from inside a Celery worker (headless/scheduled
|
||||
@@ -199,7 +199,7 @@ agent) is served by a separate worker and never self-deadlocks the awaiting one.
|
||||
Run a dedicated parsing worker that consumes the `parsing` queue:
|
||||
|
||||
```bash
|
||||
celery -A application.app.celery worker -Q parsing -l INFO
|
||||
celery -A docsgpt.app.celery worker -Q parsing -l INFO
|
||||
```
|
||||
|
||||
It takes its own env, so parse-heavy work runs on a separate, optionally larger
|
||||
@@ -214,7 +214,7 @@ and leaves this worker light.
|
||||
worker must also consume `parsing`, or the tool's await never resolves:
|
||||
|
||||
```bash
|
||||
celery -A application.app.celery worker -Q docsgpt,parsing,embeddings -l INFO
|
||||
celery -A docsgpt.app.celery worker -Q docsgpt,parsing,embeddings -l INFO
|
||||
```
|
||||
|
||||
Tuning settings: `DOCUMENT_PARSE_TIMEOUT` (seconds the tool awaits before
|
||||
|
||||
@@ -44,7 +44,7 @@ The main set of instructions or system [prompt](/Guides/Customising-prompts) tha
|
||||
|
||||
## Understanding Agent Types
|
||||
|
||||
DocsGPT supports several agent types, each with a distinct way of processing information. The code for these can be found in the `application/agents/` directory.
|
||||
DocsGPT supports several agent types, each with a distinct way of processing information. The code for these can be found in the `docsgpt/agents/` directory.
|
||||
|
||||
### 1. Classic Agent
|
||||
|
||||
@@ -117,8 +117,8 @@ Once an agent is created, you can:
|
||||
|
||||
You can bootstrap a fresh DocsGPT deployment with a curated set of agents by seeding them directly into the user-data store (Postgres).
|
||||
|
||||
1. **Customize the configuration** – edit `application/seed/config/premade_agents.yaml` (or copy from `application/seed/config/agents_template.yaml`) to describe the agents you want to provision. Each entry lets you define prompts, tools, and optional data sources.
|
||||
1. **Customize the configuration** – edit `docsgpt/seed/config/premade_agents.yaml` (or copy from `docsgpt/seed/config/agents_template.yaml`) to describe the agents you want to provision. Each entry lets you define prompts, tools, and optional data sources.
|
||||
2. **Ensure dependencies are running** – Postgres must be reachable using `POSTGRES_URI` from `.env` (schema applied via `python scripts/db/init_postgres.py`), and a Celery worker should be available if any agent sources need to be ingested via `ingest_remote`.
|
||||
3. **Execute the seeder** – run `python -m application.seed.commands init`. Add `--force` when you need to reseed an existing environment.
|
||||
3. **Execute the seeder** – run `python -m docsgpt.seed.commands init`. Add `--force` when you need to reseed an existing environment.
|
||||
|
||||
The seeder keeps templates under the `system` user so they appear in the UI for anyone to clone or customize. Environment variable placeholders such as `${MY_TOKEN}` inside tool configs are resolved during the seeding process.
|
||||
@@ -70,7 +70,7 @@ GET /api/messages/<message_id>/events
|
||||
This replays the message's events past your last-seen sequence number and tails the rest live. It is backed by the Postgres `message_events` journal (retained for `MESSAGE_EVENTS_RETENTION_DAYS`, default 14).
|
||||
|
||||
<Callout type="warning" emoji="⚠️">
|
||||
The chat reconnect endpoint is a native-async route served by the ASGI entrypoint. Under a plain `flask run` dev server it returns `404`; run the backend via the ASGI app (`uvicorn application.asgi:asgi_app`) or the production gunicorn uvicorn worker to use it. See the [Development Environment](/Deploying/Development-Environment) guide.
|
||||
The chat reconnect endpoint is a native-async route served by the ASGI entrypoint. Under a plain `flask run` dev server it returns `404`; run the backend via the ASGI app (`uvicorn docsgpt.asgi:asgi_app`) or the production gunicorn uvicorn worker to use it. See the [Development Environment](/Deploying/Development-Environment) guide.
|
||||
</Callout>
|
||||
|
||||
## Settings
|
||||
|
||||
@@ -53,7 +53,7 @@ To run the DocsGPT backend locally, you'll need to set up a Python environment a
|
||||
|
||||
* **Option 1: Using a `.env` file (Recommended):**
|
||||
* If you haven't already, create a file named `.env` in the **root directory** of your DocsGPT project.
|
||||
* Modify the `.env` file to adjust settings as needed. You can find a comprehensive list of configurable options in [`application/core/settings.py`](https://github.com/arc53/DocsGPT/blob/main/application/core/settings.py).
|
||||
* Modify the `.env` file to adjust settings as needed. You can find a comprehensive list of configurable options in [`docsgpt/core/settings.py`](https://github.com/arc53/DocsGPT/blob/main/docsgpt/core/settings.py).
|
||||
|
||||
* **Option 2: Exporting Environment Variables:**
|
||||
* Alternatively, you can export environment variables directly in your terminal. However, using a `.env` file is generally more organized for development.
|
||||
@@ -83,7 +83,7 @@ To run the DocsGPT backend locally, you'll need to set up a Python environment a
|
||||
For an offline or air-gapped machine, fetch it ahead of time instead:
|
||||
|
||||
```bash
|
||||
python -m application.scripts.prefetch_models
|
||||
python -m docsgpt.scripts.prefetch_models
|
||||
```
|
||||
|
||||
4. **Install Backend Dependencies:**
|
||||
@@ -91,40 +91,56 @@ To run the DocsGPT backend locally, you'll need to set up a Python environment a
|
||||
Navigate to the root of your DocsGPT repository and install the required Python packages:
|
||||
|
||||
```bash
|
||||
pip install -r application/requirements.txt
|
||||
pip install -r docsgpt/requirements.txt
|
||||
```
|
||||
|
||||
Optionally add the docling parser engine (OCR, `structured` output for
|
||||
`read_document`; the default anydoc engine does not need it):
|
||||
Dependencies are declared in `pyproject.toml` and locked in `uv.lock`; the
|
||||
`requirements*.txt` files are exported from that lock, so with
|
||||
[uv](https://docs.astral.sh/uv/) installed `uv sync` sets up the same
|
||||
environment (plus the test tools) in one step.
|
||||
|
||||
Optional extras are not installed by default. Add them when you need the
|
||||
feature (each file is the core set plus the extra):
|
||||
|
||||
```bash
|
||||
pip install -r application/requirements-docling.txt
|
||||
pip install -r docsgpt/requirements-docling.txt # docling parser engine: OCR backend, read_document structured output
|
||||
pip install -r docsgpt/requirements-milvus.txt # VECTOR_STORE=milvus
|
||||
# or with uv: uv sync --extra docling --extra milvus
|
||||
```
|
||||
|
||||
The docling file adds the PyTorch CPU index; pip handles that as-is, while
|
||||
`uv pip install -r` needs `UV_INDEX_STRATEGY=unsafe-best-match` (or use
|
||||
`uv sync --extra docling`, which reads the lock).
|
||||
|
||||
A feature whose extra is missing fails with the exact command to run.
|
||||
After changing `pyproject.toml`, run `uv lock` and
|
||||
`bash scripts/export_requirements.sh` so the exported files stay in sync
|
||||
(CI checks this).
|
||||
|
||||
5. **Run the Backend:**
|
||||
|
||||
For local development, run the ASGI composition under uvicorn. It serves the **whole** application, hot-reloads on source changes, and matches the production runtime:
|
||||
|
||||
```bash
|
||||
uvicorn application.asgi:asgi_app --host 0.0.0.0 --port 7091 --reload
|
||||
uvicorn docsgpt.asgi:asgi_app --host 0.0.0.0 --port 7091 --reload
|
||||
```
|
||||
|
||||
This makes the backend accessible on `http://localhost:7091`. Production uses `gunicorn -k uvicorn_worker.UvicornWorker` against the same `application.asgi:asgi_app` target.
|
||||
This makes the backend accessible on `http://localhost:7091`. Production uses `gunicorn -k uvicorn_worker.UvicornWorker` against the same `docsgpt.asgi:asgi_app` target.
|
||||
|
||||
A plain Flask run is a faster inner loop (quick startup, the Werkzeug interactive debugger):
|
||||
|
||||
```bash
|
||||
flask --app application/app.py run --host=0.0.0.0 --port=7091
|
||||
flask --app docsgpt/app.py run --host=0.0.0.0 --port=7091
|
||||
```
|
||||
|
||||
But it serves **only** the WSGI Flask app and omits the routes mounted on the ASGI shell in `application/asgi.py`: the `/mcp` FastMCP endpoint and the native-async SSE reconnect reader `GET /api/messages/<id>/events`. Under `flask run` those paths return 404 — chat still works (`POST /stream` is a Flask route), but a stream interrupted by a disconnect won't auto-resume on reconnect. Use `flask run` only when you don't need those routes.
|
||||
But it serves **only** the WSGI Flask app and omits the routes mounted on the ASGI shell in `docsgpt/asgi.py`: the `/mcp` FastMCP endpoint and the native-async SSE reconnect reader `GET /api/messages/<id>/events`. Under `flask run` those paths return 404 — chat still works (`POST /stream` is a Flask route), but a stream interrupted by a disconnect won't auto-resume on reconnect. Use `flask run` only when you don't need those routes.
|
||||
|
||||
6. **Start the Celery Worker:**
|
||||
|
||||
Open a new terminal window (and activate your virtual environment if you used one). Start the Celery worker to handle background tasks:
|
||||
|
||||
```bash
|
||||
celery -A application.app.celery worker -l INFO
|
||||
celery -A docsgpt.app.celery worker -l INFO
|
||||
```
|
||||
|
||||
This command will start the Celery worker, which processes tasks such as document parsing and vector embedding.
|
||||
@@ -132,7 +148,7 @@ To run the DocsGPT backend locally, you'll need to set up a Python environment a
|
||||
**macOS note:** Due to a threading issue, start Celery with the solo pool:
|
||||
|
||||
```bash
|
||||
python -m celery -A application.app.celery worker -l INFO --pool=solo
|
||||
python -m celery -A docsgpt.app.celery worker -l INFO --pool=solo
|
||||
```
|
||||
|
||||
**Running in Debugger (VSCode):**
|
||||
|
||||
@@ -17,9 +17,62 @@ Docker is the recommended method for deploying DocsGPT, providing a consistent a
|
||||
|
||||
**Important Note for Windows Users:** Docker Desktop on Windows generally requires the WSL 2 backend to function correctly, especially when using features like host networking which are utilized in DocsGPT's Docker Compose setup. Ensure WSL 2 is enabled and configured in Docker Desktop settings.
|
||||
|
||||
## Quickest Setup: Using DocsGPT Public API
|
||||
## Quickest Setup: Pre-built Images, No Checkout
|
||||
|
||||
The fastest way to try out DocsGPT is by using the public API endpoint. This requires minimal configuration and no local LLM setup.
|
||||
Every release publishes ready-to-run images to Docker Hub (`arc53/docsgpt`,
|
||||
`arc53/docsgpt-fe`) and GitHub Container Registry (`ghcr.io/arc53/docsgpt`,
|
||||
`ghcr.io/arc53/docsgpt-fe`) for `linux/amd64` and `linux/arm64`. The images
|
||||
contain everything the default configuration needs (embedding models,
|
||||
tokenizers, tiktoken's encoding), so a fresh container makes no downloads on
|
||||
first use. You do not need the source tree to run them:
|
||||
|
||||
1. **Download the standalone Compose file** (also attached to every
|
||||
[release](https://github.com/arc53/DocsGPT/releases)):
|
||||
|
||||
```bash
|
||||
mkdir docsgpt && cd docsgpt
|
||||
curl -fsSLO https://raw.githubusercontent.com/arc53/DocsGPT/main/deployment/docker-compose-standalone.yaml
|
||||
```
|
||||
|
||||
2. **Create a `.env` next to it** with your settings, for example the public API:
|
||||
|
||||
```bash
|
||||
printf 'LLM_PROVIDER=docsgpt\nVITE_API_STREAMING=true\nINTERNAL_KEY=%s\n' "$(openssl rand -hex 16)" > .env
|
||||
```
|
||||
|
||||
`INTERNAL_KEY` is the secret the worker uses to hand finished indexes to
|
||||
the API; without it every upload fails with a 401. `setup.sh` generates
|
||||
one for you, a hand-written `.env` has to include it. This stack runs the
|
||||
granite embedding model unless `.env` sets `EMBEDDINGS_NAME`; both granite
|
||||
and mpnet are baked into the image.
|
||||
|
||||
3. **Start it:**
|
||||
|
||||
```bash
|
||||
docker compose -f docker-compose-standalone.yaml up -d
|
||||
```
|
||||
|
||||
Then open [http://localhost:5173/](http://localhost:5173/). Data lives in
|
||||
named Docker volumes; `docker compose -f docker-compose-standalone.yaml down`
|
||||
keeps it and `down -v` removes it.
|
||||
|
||||
**Tags and variants.** `DOCSGPT_IMAGE_TAG` picks the version: a release such
|
||||
as `0.20.0`, `latest` (the newest release, the default) or `develop` (follows
|
||||
the `main` branch). `DOCSGPT_IMAGE_VARIANT` picks the flavour: empty for the
|
||||
slim default image, or `-docling` for the image with the docling parser
|
||||
engine, its models and tesseract baked in (needed for OCR of scanned
|
||||
documents, see the [OCR guide](/Guides/ocr)). Both are read from `.env` or
|
||||
the shell, e.g. `DOCSGPT_IMAGE_TAG=0.20.0 DOCSGPT_IMAGE_VARIANT=-docling`.
|
||||
The same two variables drive `deployment/docker-compose-hub.yaml` in a
|
||||
checkout.
|
||||
|
||||
## Using the Source Checkout
|
||||
|
||||
With a clone of the repository, `deployment/docker-compose-hub.yaml` runs the
|
||||
same pre-built images while keeping your data in `application/indexes`,
|
||||
`application/inputs` and `application/vectors`, and `deployment/docker-compose.yaml`
|
||||
builds the images from your working tree (for local changes, or a build with
|
||||
extra packages: `EXTRAS=docling` in `.env`).
|
||||
|
||||
1. **Clone the DocsGPT Repository (if you haven't already):**
|
||||
|
||||
@@ -39,19 +92,26 @@ The fastest way to try out DocsGPT is by using the public API endpoint. This req
|
||||
```
|
||||
LLM_PROVIDER=docsgpt
|
||||
VITE_API_STREAMING=true
|
||||
INTERNAL_KEY=<any random string, e.g. openssl rand -hex 16>
|
||||
EMBEDDINGS_NAME=ibm-granite/granite-embedding-311m-multilingual-r2
|
||||
```
|
||||
|
||||
This minimal configuration tells DocsGPT to use the public API. For more advanced settings and other LLM options, refer to the [DocsGPT Settings Guide](/Deploying/DocsGPT-Settings).
|
||||
This minimal configuration tells DocsGPT to use the public API. The
|
||||
`EMBEDDINGS_NAME` line is what `setup.sh` writes for a new install; without
|
||||
it the code falls back to mpnet, the model earlier releases indexed with,
|
||||
so that an upgraded deployment keeps its existing index working. For more advanced settings and other LLM options, refer to the [DocsGPT Settings Guide](/Deploying/DocsGPT-Settings).
|
||||
|
||||
4. **Launch DocsGPT with Docker Compose:**
|
||||
|
||||
Navigate to the root directory of the DocsGPT repository in your terminal and run:
|
||||
|
||||
```bash
|
||||
docker compose --env-file .env -f deployment/docker-compose.yaml up -d
|
||||
docker compose --env-file .env -f deployment/docker-compose-hub.yaml up -d
|
||||
```
|
||||
|
||||
The `-d` flag runs Docker Compose in detached mode (in the background).
|
||||
To build the images from your working tree instead of pulling them, use
|
||||
`deployment/docker-compose.yaml` with `up --build -d`.
|
||||
|
||||
5. **Access DocsGPT in your browser:**
|
||||
|
||||
@@ -62,7 +122,7 @@ The fastest way to try out DocsGPT is by using the public API endpoint. This req
|
||||
To stop the application, navigate to the same directory in your terminal and run:
|
||||
|
||||
```bash
|
||||
docker compose -f deployment/docker-compose.yaml down
|
||||
docker compose -f deployment/docker-compose-hub.yaml down
|
||||
```
|
||||
|
||||
## Optional Ollama Setup (Local Models)
|
||||
@@ -84,11 +144,11 @@ There are two Ollama optional files:
|
||||
|
||||
**CPU:**
|
||||
```bash
|
||||
docker compose --env-file .env -f deployment/docker-compose.yaml -f deployment/optional/docker-compose.optional.ollama-cpu.yaml up -d
|
||||
docker compose --env-file .env -f deployment/docker-compose-hub.yaml -f deployment/optional/docker-compose.optional.ollama-cpu.yaml up -d
|
||||
```
|
||||
**GPU:**
|
||||
```bash
|
||||
docker compose --env-file .env -f deployment/docker-compose.yaml -f deployment/optional/docker-compose.optional.ollama-gpu.yaml up -d
|
||||
docker compose --env-file .env -f deployment/docker-compose-hub.yaml -f deployment/optional/docker-compose.optional.ollama-gpu.yaml up -d
|
||||
```
|
||||
|
||||
3. **Pull the Ollama Model:**
|
||||
@@ -96,11 +156,11 @@ There are two Ollama optional files:
|
||||
**Crucially, after launching with Ollama, you need to pull the desired model into the Ollama container.** Find the `LLM_NAME` you configured in your `.env` file (e.g., `llama3.2:1b`). Then execute the following command to pull the model *inside* the running Ollama container:
|
||||
|
||||
```bash
|
||||
docker compose -f deployment/docker-compose.yaml -f deployment/optional/docker-compose.optional.ollama-cpu.yaml exec -it ollama ollama pull <LLM_NAME>
|
||||
docker compose -f deployment/docker-compose-hub.yaml -f deployment/optional/docker-compose.optional.ollama-cpu.yaml exec -it ollama ollama pull <LLM_NAME>
|
||||
```
|
||||
or (for GPU):
|
||||
```bash
|
||||
docker compose -f deployment/docker-compose.yaml -f deployment/optional/docker-compose.optional.ollama-gpu.yaml exec -it ollama ollama pull <LLM_NAME>
|
||||
docker compose -f deployment/docker-compose-hub.yaml -f deployment/optional/docker-compose.optional.ollama-gpu.yaml exec -it ollama ollama pull <LLM_NAME>
|
||||
```
|
||||
Replace `<LLM_NAME>` with the actual model name from your `.env` file.
|
||||
|
||||
@@ -113,12 +173,12 @@ There are two Ollama optional files:
|
||||
To stop a DocsGPT setup launched with Ollama optional files, use `docker compose down` and include all the compose files used during the `up` command:
|
||||
|
||||
```bash
|
||||
docker compose -f deployment/docker-compose.yaml -f deployment/optional/docker-compose.optional.ollama-cpu.yaml down
|
||||
docker compose -f deployment/docker-compose-hub.yaml -f deployment/optional/docker-compose.optional.ollama-cpu.yaml down
|
||||
```
|
||||
or
|
||||
|
||||
```bash
|
||||
docker compose -f deployment/docker-compose.yaml -f deployment/optional/docker-compose.optional.ollama-gpu.yaml down
|
||||
docker compose -f deployment/docker-compose-hub.yaml -f deployment/optional/docker-compose.optional.ollama-gpu.yaml down
|
||||
```
|
||||
|
||||
**Important for GPU Usage:**
|
||||
|
||||
@@ -29,11 +29,11 @@ LLM_NAME=gpt-4o
|
||||
|
||||
### 2. Configuration via `settings.py` file (Advanced)
|
||||
|
||||
For more advanced configurations or if you prefer to manage settings directly in code, you can modify the `settings.py` file. This file is located in the `application/core` directory of your DocsGPT project.
|
||||
For more advanced configurations or if you prefer to manage settings directly in code, you can modify the `settings.py` file. This file is located in the `docsgpt/core` directory of your DocsGPT project.
|
||||
|
||||
While modifying `settings.py` offers more flexibility, it's generally recommended to use the `.env` file for basic settings and reserve `settings.py` for more complex adjustments or when you need to configure settings programmatically.
|
||||
|
||||
**Location of `settings.py`:** `application/core/settings.py`
|
||||
**Location of `settings.py`:** `docsgpt/core/settings.py`
|
||||
|
||||
## Basic Settings Explained
|
||||
|
||||
@@ -61,7 +61,7 @@ Here are some of the most fundamental settings you'll likely want to configure:
|
||||
|
||||
- **Default value:** leave it unset and DocsGPT picks for you at first boot, recording the choice so it never changes underneath you: a fresh install is pinned to `ibm-granite/granite-embedding-311m-multilingual-r2` (multilingual, 32k context, same 768 dimensions), and an install that already has sources is pinned to `huggingface_sentence-transformers/all-mpnet-base-v2` so its index stays readable. Setting it here overrides that pin.
|
||||
- **Other options:** Any FastEmbed built-in model, or any Hugging Face repository shipping an ONNX export. See [Embeddings](/Models/embeddings).
|
||||
- **Changing it on an existing index requires re-embedding** — same-width models swap without any error and silently degrade retrieval. Run `python -m application.scripts.reembed`.
|
||||
- **Changing it on an existing index requires re-embedding** — same-width models swap without any error and silently degrade retrieval. Run `python -m docsgpt.scripts.reembed`.
|
||||
|
||||
- **`API_KEY`**: Required for most cloud-based LLM providers. This is your authentication key to access the LLM provider's API. You'll need to obtain this key from your chosen provider's platform.
|
||||
|
||||
@@ -133,7 +133,7 @@ models:
|
||||
|
||||
After restart, those models appear in `/api/models` and are selectable
|
||||
in the UI. A working template lives at
|
||||
`application/core/models/examples/mistral.yaml.example`.
|
||||
`docsgpt/core/models/examples/mistral.yaml.example`.
|
||||
|
||||
**What you can do:**
|
||||
|
||||
@@ -148,8 +148,8 @@ in the UI. A working template lives at
|
||||
|
||||
**What you cannot do via `MODELS_CONFIG_DIR`:** add a brand-new
|
||||
non-OpenAI provider. That requires a Python plugin under
|
||||
`application/llm/providers/`. See
|
||||
`application/core/models/README.md` for the full schema reference.
|
||||
`docsgpt/llm/providers/`. See
|
||||
`docsgpt/core/models/README.md` for the full schema reference.
|
||||
|
||||
### Docker
|
||||
|
||||
@@ -207,7 +207,7 @@ for the engines and flows.
|
||||
|
||||
| Setting | Default | Description |
|
||||
| --- | --- | --- |
|
||||
| `DOC_PARSER_ENGINE` | `anydoc` | Parser engine: `anydoc` (fast Rust converter, no ML models) or `docling` (layout/table models, structured output). Files anydoc cannot read fall back to Docling when installed. Docling is an optional extra: `pip install -r application/requirements-docling.txt`, or Docker builds with `--build-arg INSTALL_DOCLING=true`. |
|
||||
| `DOC_PARSER_ENGINE` | `anydoc` | Parser engine: `anydoc` (fast Rust converter, no ML models) or `docling` (layout/table models, structured output). Files anydoc cannot read fall back to Docling when installed. Docling is an optional extra: `pip install -r docsgpt/requirements-docling.txt`, or Docker builds with `--build-arg INSTALL_DOCLING=true`. |
|
||||
| `OCR_ENABLED` | `false` | OCR for source ingestion. Alias: `DOCLING_OCR_ENABLED`. |
|
||||
| `OCR_ATTACHMENTS_ENABLED` | `false` | OCR for chat attachments. Alias: `DOCLING_OCR_ATTACHMENTS_ENABLED`. |
|
||||
| `OCR_BACKEND` | `auto` | Who performs OCR: `auto` (Docling when installed, else native), `native` (pypdfium2 + Pillow rendering into tesseract or DeepSeek-OCR; no docling needed), or `docling` (layout-model hybrid OCR). See the [OCR guide](/Guides/ocr#ocr-backends). |
|
||||
@@ -469,8 +469,9 @@ See [Embeddings](/Models/embeddings) for full guidance.
|
||||
|
||||
| Setting | Default | Description |
|
||||
| --- | --- | --- |
|
||||
| `EMBEDDINGS_NAME` | `huggingface_sentence-transformers/all-mpnet-base-v2` | The embedding model. New installs use `ibm-granite/granite-embedding-311m-multilingual-r2`. Changing it on a populated index requires `application.scripts.reembed`. |
|
||||
| `EMBEDDINGS_NAME` | `huggingface_sentence-transformers/all-mpnet-base-v2` | The embedding model. New installs use `ibm-granite/granite-embedding-311m-multilingual-r2`. Changing it on a populated index requires `docsgpt.scripts.reembed`. |
|
||||
| `EMBEDDINGS_BASE_URL` | unset | Base URL of a remote OpenAI-compatible embeddings server. Setting it routes all embedding calls there. |
|
||||
| `EMBEDDINGS_THREADS` | unset (Docker image: `4`) | Threads one local FastEmbed/onnxruntime session may use. onnxruntime otherwise sizes its pool to the host's core count, which a CPU-limited container still reports, so the image pins it like `OMP_NUM_THREADS`. Raise it on a large dedicated worker. |
|
||||
| `EMBEDDINGS_KEY` | unset | Optional bearer token for the remote embeddings server. |
|
||||
| `EMBEDDINGS_MAX_INPUT_TOKENS` | unset | Truncate each remote embedding input to N tokens (guards servers that reject oversized inputs). |
|
||||
| `EMBEDDINGS_DELEGATE_TO_WORKER` | `true` | Embed queries on the Celery worker instead of loading a model in the API. Requires a worker consuming `EMBEDDINGS_QUEUE`; set `false` to run the API standalone. Ignored when `EMBEDDINGS_BASE_URL` is set. |
|
||||
@@ -549,4 +550,4 @@ These are just the basic settings to get you started. The `settings.py` file con
|
||||
- Cache settings (`CACHE_REDIS_URL`)
|
||||
- And many more!
|
||||
|
||||
For a complete list of available settings and their descriptions, refer to the `settings.py` file in `application/core`. Remember to restart your Docker containers after making changes to your `.env` file or `settings.py` for the changes to take effect.
|
||||
For a complete list of available settings and their descriptions, refer to the `settings.py` file in `docsgpt/core`. Remember to restart your Docker containers after making changes to your `.env` file or `settings.py` for the changes to take effect.
|
||||
@@ -8,7 +8,7 @@ import { Callout } from 'nextra/components'
|
||||
# Observability
|
||||
|
||||
DocsGPT bundles the OpenTelemetry SDK and auto-instrumentation packages
|
||||
in `application/requirements.txt` — they install with the rest of the
|
||||
in `docsgpt/requirements.txt` — they install with the rest of the
|
||||
backend deps. Telemetry is **off by default**; opt in by prefixing the
|
||||
launch command with `opentelemetry-instrument` and setting OTLP env
|
||||
vars.
|
||||
@@ -42,12 +42,12 @@ services:
|
||||
backend:
|
||||
command: >
|
||||
opentelemetry-instrument gunicorn -w 1 -k uvicorn_worker.UvicornWorker
|
||||
--bind 0.0.0.0:7091 --config application/gunicorn_conf.py
|
||||
application.asgi:asgi_app
|
||||
--bind 0.0.0.0:7091 --config docsgpt/gunicorn_conf.py
|
||||
docsgpt.asgi:asgi_app
|
||||
environment:
|
||||
- OTEL_SERVICE_NAME=docsgpt-backend
|
||||
worker:
|
||||
command: opentelemetry-instrument celery -A application.app.celery worker -l INFO -B
|
||||
command: opentelemetry-instrument celery -A docsgpt.app.celery worker -l INFO -B
|
||||
environment:
|
||||
- OTEL_SERVICE_NAME=docsgpt-celery-worker
|
||||
```
|
||||
@@ -56,14 +56,14 @@ For local dev, prepend `dotenv run --` so the `OTEL_*` vars from `.env`
|
||||
reach `opentelemetry-instrument` before it boots the SDK:
|
||||
|
||||
```bash
|
||||
dotenv run -- opentelemetry-instrument flask --app application/app.py run --port=7091
|
||||
dotenv run -- opentelemetry-instrument celery -A application.app.celery worker -l INFO --pool=solo
|
||||
dotenv run -- opentelemetry-instrument flask --app docsgpt/app.py run --port=7091
|
||||
dotenv run -- opentelemetry-instrument celery -A docsgpt.app.celery worker -l INFO --pool=solo
|
||||
```
|
||||
|
||||
|
||||
<Callout type="info" emoji="ℹ️">
|
||||
Logs are exported in-process when `OTEL_LOGS_EXPORTER=otlp` is set —
|
||||
`application/core/logging_config.py` detects the flag and preserves
|
||||
`docsgpt/core/logging_config.py` detects the flag and preserves
|
||||
the OTEL log handler. Without it, `logging` writes only to stdout.
|
||||
</Callout>
|
||||
|
||||
|
||||
@@ -37,7 +37,7 @@ schema on first boot.
|
||||
|
||||
```bash
|
||||
export POSTGRES_URI="postgresql://user:pass@host/docsgpt?sslmode=require"
|
||||
uvicorn application.asgi:asgi_app --host 0.0.0.0 --port 7091
|
||||
uvicorn docsgpt.asgi:asgi_app --host 0.0.0.0 --port 7091
|
||||
```
|
||||
|
||||
### Bare-metal Postgres
|
||||
@@ -47,7 +47,7 @@ First boot creates both the database and the schema.
|
||||
|
||||
```bash
|
||||
export POSTGRES_URI="postgresql://postgres@localhost/docsgpt"
|
||||
uvicorn application.asgi:asgi_app --host 0.0.0.0 --port 7091
|
||||
uvicorn docsgpt.asgi:asgi_app --host 0.0.0.0 --port 7091
|
||||
```
|
||||
|
||||
Prefer a dedicated non-superuser role? Create it once as superuser — the
|
||||
@@ -93,7 +93,7 @@ init-container ahead of the app rollout:
|
||||
```bash
|
||||
python scripts/db/init_postgres.py
|
||||
# equivalently:
|
||||
alembic -c application/alembic.ini upgrade head
|
||||
alembic -c docsgpt/alembic.ini upgrade head
|
||||
```
|
||||
|
||||
The reasoning: the app's runtime role shouldn't carry DDL privileges,
|
||||
@@ -112,7 +112,7 @@ One-shot, offline, app stopped. The app itself will create the
|
||||
Postgres schema when it boots — you only need to run the data copy.
|
||||
|
||||
```bash
|
||||
pip install -r application/requirements.txt
|
||||
pip install -r docsgpt/requirements.txt
|
||||
pip install 'pymongo>=4.6'
|
||||
|
||||
export POSTGRES_URI="postgresql://docsgpt:docsgpt@localhost:5432/docsgpt"
|
||||
|
||||
@@ -359,7 +359,7 @@ Technical documentation about...
|
||||
### Template Validation
|
||||
Test your template syntax before saving:
|
||||
```python
|
||||
from application.api.answer.services.prompt_renderer import PromptRenderer
|
||||
from docsgpt.api.answer.services.prompt_renderer import PromptRenderer
|
||||
|
||||
renderer = PromptRenderer()
|
||||
is_valid = renderer.validate_template("Your prompt with {{ variables }}")
|
||||
@@ -487,7 +487,7 @@ Provide detailed answers appropriate for {{ passthrough.access_level }} access l
|
||||
|
||||
### Render Prompt via API
|
||||
```python
|
||||
from application.api.answer.services.prompt_renderer import PromptRenderer
|
||||
from docsgpt.api.answer.services.prompt_renderer import PromptRenderer
|
||||
|
||||
renderer = PromptRenderer()
|
||||
rendered = renderer.render_prompt(
|
||||
|
||||
@@ -19,7 +19,7 @@ The compression system operates on a "summarize and truncate" principle:
|
||||
|
||||
## Configuration
|
||||
|
||||
You can configure the compression behavior in your `.env` file or `application/core/settings.py`:
|
||||
You can configure the compression behavior in your `.env` file or `docsgpt/core/settings.py`:
|
||||
|
||||
| Setting | Default | Description |
|
||||
| :--- | :--- | :--- |
|
||||
|
||||
+28
-14
@@ -57,18 +57,24 @@ apt-get install tesseract-ocr tesseract-ocr-eng brew install tesseract
|
||||
Docker images build without it by default; opt in with the build argument:
|
||||
|
||||
```bash
|
||||
docker build --build-arg INSTALL_TESSERACT=true ./application
|
||||
docker build -f docsgpt/Dockerfile --build-arg INSTALL_TESSERACT=true .
|
||||
```
|
||||
|
||||
`deployment/docker-compose.yaml` forwards the same switch, so setting
|
||||
`INSTALL_TESSERACT=true` in `.env` (or the shell) bakes tesseract plus the
|
||||
English pack into locally built backend and worker images; `setup.sh` writes
|
||||
it when you answer yes to the OCR question after choosing to build images
|
||||
locally. Pre-built Docker Hub images (`docker-compose-hub.yaml`) do not
|
||||
include it, so `setup.sh` leaves OCR at its default (off) for them; to OCR
|
||||
there, point `OCR_ENGINE=deepseek` at a DeepSeek-OCR endpoint or install
|
||||
`tesseract-ocr` in a derived image. With `OCR_ENABLED=true` and no binary on
|
||||
`PATH`, scanned pages fail with an install hint (text-layer documents are
|
||||
locally.
|
||||
|
||||
With pre-built images the switch is the image variant: every tag is
|
||||
published twice, slim (`arc53/docsgpt:<tag>`) and `-docling`
|
||||
(`arc53/docsgpt:<tag>-docling`), and the latter bakes tesseract, the docling
|
||||
engine and its models in. Set `DOCSGPT_IMAGE_VARIANT=-docling` in `.env` for
|
||||
`docker-compose-hub.yaml` or `docker-compose-standalone.yaml`; `setup.sh`
|
||||
writes it when you answer yes to the OCR question with Docker Hub images.
|
||||
Alternatively point `OCR_ENGINE=deepseek` at a DeepSeek-OCR endpoint, which
|
||||
needs no system package. With `OCR_ENABLED=true` and no binary on `PATH`,
|
||||
scanned pages fail with an install hint (text-layer documents are
|
||||
unaffected).
|
||||
|
||||
<Callout type="warning" emoji="⚠️">
|
||||
@@ -88,23 +94,31 @@ docling is not part of the base install, and OCR does not need it (see
|
||||
output:
|
||||
|
||||
```bash
|
||||
pip install -r application/requirements-docling.txt
|
||||
pip install -r docsgpt/requirements-docling.txt # or: uv sync --extra docling
|
||||
```
|
||||
|
||||
Docker images build without it by default; opt in with the build argument:
|
||||
That file is the core set plus the `docling` extra, exported from the same
|
||||
lock. On Linux it takes torch from the CPU-only PyTorch index, so the extra
|
||||
costs about 1.5 GB rather than the 2.7 GB the CUDA build of torch would; a
|
||||
GPU deployment can reinstall torch from PyPI on top.
|
||||
|
||||
Pre-built images: use the `-docling` variant (`arc53/docsgpt:<tag>-docling`,
|
||||
`DOCSGPT_IMAGE_VARIANT=-docling` in `.env`), which also bakes docling's
|
||||
layout, table-structure and RapidOCR models in so the first parse does not
|
||||
download them. Local builds opt in with the build argument:
|
||||
|
||||
```bash
|
||||
docker build --build-arg INSTALL_DOCLING=true ./application
|
||||
docker build -f docsgpt/Dockerfile --build-arg EXTRAS=docling .
|
||||
```
|
||||
|
||||
`deployment/docker-compose.yaml` forwards the same switch, so setting
|
||||
`INSTALL_DOCLING=true` in `.env` (or the shell) bakes docling into locally
|
||||
built backend and worker images; `setup.sh` offers it as a follow-up to the
|
||||
OCR question. Compose reads build arguments from the shell or from the
|
||||
`EXTRAS=docling` (or the older `INSTALL_DOCLING=true`) in `.env` (or the
|
||||
shell) bakes docling into locally built backend and worker images; `setup.sh`
|
||||
offers it as a follow-up to the OCR question. Compose reads build arguments from the shell or from the
|
||||
`.env` you pass with `--env-file .env` (not from the containers' `env_file`),
|
||||
so build with `docker compose --env-file .env -f deployment/docker-compose.yaml build`
|
||||
as `setup.sh` does. Pre-built Docker Hub images (`docker-compose-hub.yaml`)
|
||||
never include docling; install it in a derived image instead. Either way no
|
||||
as `setup.sh` does. Of the pre-built Docker Hub images only the slim default
|
||||
excludes docling; the `-docling` variant ships it with its models. Either way no
|
||||
code changes are needed — docling is picked up as
|
||||
the fallback engine (and, under `OCR_BACKEND=auto`, as the OCR backend) as
|
||||
soon as it is importable, and `DOC_PARSER_ENGINE=docling` makes it the
|
||||
|
||||
@@ -33,7 +33,7 @@ DocsGPT offers direct, streamlined support for the following cloud LLM providers
|
||||
| Novita AI | `novita` | (See Novita docs) |
|
||||
| HuggingFace Inference API | `huggingface` | `meta-llama/Llama-3.1-8B-Instruct` |
|
||||
|
||||
DocsGPT also ships a **model catalog** (`application/core/models/*.yaml`) that the in-app model picker reads, so common models from these providers — including DeepSeek — appear ready to select once the matching API key is set.
|
||||
DocsGPT also ships a **model catalog** (`docsgpt/core/models/*.yaml`) that the in-app model picker reads, so common models from these providers — including DeepSeek — appear ready to select once the matching API key is set.
|
||||
|
||||
## Connecting to OpenAI-Compatible Cloud APIs
|
||||
|
||||
@@ -75,4 +75,4 @@ See [App Configuration](/Deploying/DocsGPT-Settings) for the full settings refer
|
||||
|
||||
## Adding Support for Other Cloud Providers
|
||||
|
||||
If you wish to connect to a cloud provider that is not explicitly listed above or doesn't offer OpenAI API compatibility, you can extend DocsGPT to support it. Within the DocsGPT repository, navigate to the `application/llm` directory. Here, you will find Python files defining the existing LLM integrations. You can use these files as examples to create a new module for your desired cloud provider. After creating your new LLM module, you will need to register it within the `llm_creator.py` file. This process involves some coding, but it allows for virtually unlimited extensibility to connect to any cloud-based LLM service with an accessible API.
|
||||
If you wish to connect to a cloud provider that is not explicitly listed above or doesn't offer OpenAI API compatibility, you can extend DocsGPT to support it. Within the DocsGPT repository, navigate to the `docsgpt/llm` directory. Here, you will find Python files defining the existing LLM integrations. You can use these files as examples to create a new module for your desired cloud provider. After creating your new LLM module, you will need to register it within the `llm_creator.py` file. This process involves some coding, but it allows for virtually unlimited extensibility to connect to any cloud-based LLM service with an accessible API.
|
||||
@@ -46,7 +46,7 @@ Models with a Dense projection layer (for example `sentence-transformers/LaBSE`)
|
||||
For an offline or air-gapped install, pre-fetch the model at build or setup time:
|
||||
|
||||
```bash
|
||||
python -m application.scripts.prefetch_models
|
||||
python -m docsgpt.scripts.prefetch_models
|
||||
```
|
||||
|
||||
## Using OpenAI Embeddings
|
||||
@@ -100,7 +100,7 @@ Retrieval then depends on a worker consuming `EMBEDDINGS_QUEUE` (`embeddings` by
|
||||
Sharing one worker also shares its concurrency with ingest, so a query can queue behind a long parse. Run a dedicated worker to isolate query latency:
|
||||
|
||||
```bash
|
||||
celery -A application.app.celery worker -Q embeddings
|
||||
celery -A docsgpt.app.celery worker -Q embeddings
|
||||
```
|
||||
|
||||
Set `EMBEDDINGS_DELEGATE_TO_WORKER=false` if you run the API without a worker; it will load the model in-process instead.
|
||||
@@ -139,7 +139,7 @@ The dimension check is a guard against a corrupt index, not a guarantee that a s
|
||||
Switching between same-width models therefore still requires re-embedding:
|
||||
|
||||
```bash
|
||||
python -m application.scripts.reembed
|
||||
python -m docsgpt.scripts.reembed
|
||||
```
|
||||
|
||||
Run it after changing `EMBEDDINGS_NAME` and before serving queries. See [Upgrading](/upgrading) for the granite migration specifically.
|
||||
@@ -150,6 +150,6 @@ With `GRAPHRAG_ENABLED`, the script also rewrites `graph_nodes.name_embedding` o
|
||||
|
||||
## Adding Support for Other Embedding Models
|
||||
|
||||
To teach DocsGPT about a new model — so it carries a known pooling, width and context window rather than being inferred — add an `EmbeddingModel` entry to `MODELS` in `application/vectorstore/model_registry.py`. That registry is the single source of truth the local runner, the remote client, the schema bootstrap and the chunker all read.
|
||||
To teach DocsGPT about a new model — so it carries a known pooling, width and context window rather than being inferred — add an `EmbeddingModel` entry to `MODELS` in `docsgpt/vectorstore/model_registry.py`. That registry is the single source of truth the local runner, the remote client, the schema bootstrap and the chunker all read.
|
||||
|
||||
Specifically, pay attention to the `EmbeddingsWrapper` and `EmbeddingsSingleton` classes. `EmbeddingsWrapper` provides a way to wrap different embedding model libraries into a consistent interface for DocsGPT. `EmbeddingsSingleton` manages the instantiation and retrieval of embedding model instances. By understanding these classes and the existing embedding model implementations, you can create your own custom integration for virtually any embedding model library you desire.
|
||||
@@ -86,4 +86,4 @@ You can configure multiple API keys simultaneously (e.g., both `OPENAI_API_KEY`
|
||||
|
||||
## Adding Support for Other Local Engines
|
||||
|
||||
While DocsGPT currently focuses on OpenAI API compatible local engines, you can extend its capabilities to support other local inference solutions. To do this, navigate to the `application/llm` directory in the DocsGPT repository. Examine the existing Python files for examples of LLM integrations. You can create a new module for your desired local engine, and then register it in the `llm_creator.py` file within the same directory. This allows for custom integration with a wide range of local LLM servers beyond those listed above.
|
||||
While DocsGPT currently focuses on OpenAI API compatible local engines, you can extend its capabilities to support other local inference solutions. To do this, navigate to the `docsgpt/llm` directory in the DocsGPT repository. Examine the existing Python files for examples of LLM integrations. You can create a new module for your desired local engine, and then register it in the `llm_creator.py` file within the same directory. This allows for custom integration with a wide range of local LLM servers beyond those listed above.
|
||||
@@ -38,37 +38,37 @@ DocsGPT includes a suite of pre-built tools designed to expand its capabilities
|
||||
},
|
||||
{
|
||||
title: 'Brave Search',
|
||||
link: 'https://github.com/arc53/DocsGPT/blob/main/application/agents/tools/brave.py',
|
||||
link: 'https://github.com/arc53/DocsGPT/blob/main/docsgpt/agents/tools/brave.py',
|
||||
description: 'Enables DocsGPT to perform real-time web and image searches using the Brave Search API. Requires an API key.'
|
||||
},
|
||||
{
|
||||
title: 'DuckDuckGo Search',
|
||||
link: 'https://github.com/arc53/DocsGPT/blob/main/application/agents/tools/duckduckgo.py',
|
||||
link: 'https://github.com/arc53/DocsGPT/blob/main/docsgpt/agents/tools/duckduckgo.py',
|
||||
description: 'Performs web and image searches using DuckDuckGo. No API key required.'
|
||||
},
|
||||
{
|
||||
title: 'CryptoPrice',
|
||||
link: 'https://github.com/arc53/DocsGPT/blob/main/application/agents/tools/cryptoprice.py',
|
||||
link: 'https://github.com/arc53/DocsGPT/blob/main/docsgpt/agents/tools/cryptoprice.py',
|
||||
description: 'Fetches the current price of specified cryptocurrencies using the CryptoCompare public API.'
|
||||
},
|
||||
{
|
||||
title: 'Ntfy',
|
||||
link: 'https://github.com/arc53/DocsGPT/blob/main/application/agents/tools/ntfy.py',
|
||||
link: 'https://github.com/arc53/DocsGPT/blob/main/docsgpt/agents/tools/ntfy.py',
|
||||
description: 'Allows DocsGPT to send push notifications to ntfy topics on a specified server, ideal for alerts and updates.'
|
||||
},
|
||||
{
|
||||
title: 'Telegram Bot',
|
||||
link: 'https://github.com/arc53/DocsGPT/blob/main/application/agents/tools/telegram.py',
|
||||
link: 'https://github.com/arc53/DocsGPT/blob/main/docsgpt/agents/tools/telegram.py',
|
||||
description: 'Allows DocsGPT to send messages or images to Telegram chats via a Telegram Bot. Requires a bot token and chat ID.'
|
||||
},
|
||||
{
|
||||
title: 'PostgreSQL Database',
|
||||
link: 'https://github.com/arc53/DocsGPT/blob/main/application/agents/tools/postgres.py',
|
||||
link: 'https://github.com/arc53/DocsGPT/blob/main/docsgpt/agents/tools/postgres.py',
|
||||
description: 'Connects to a PostgreSQL database to execute SQL queries and retrieve schema information.'
|
||||
},
|
||||
{
|
||||
title: 'Read Webpage (browser)',
|
||||
link: 'https://github.com/arc53/DocsGPT/blob/main/application/agents/tools/read_webpage.py',
|
||||
link: 'https://github.com/arc53/DocsGPT/blob/main/docsgpt/agents/tools/read_webpage.py',
|
||||
description: 'Fetches the HTML content of a URL and converts it to Markdown for the agent to read.'
|
||||
},
|
||||
{
|
||||
@@ -83,17 +83,17 @@ DocsGPT includes a suite of pre-built tools designed to expand its capabilities
|
||||
},
|
||||
{
|
||||
title: 'Memory',
|
||||
link: 'https://github.com/arc53/DocsGPT/blob/main/application/agents/tools/memory.py',
|
||||
link: 'https://github.com/arc53/DocsGPT/blob/main/docsgpt/agents/tools/memory.py',
|
||||
description: 'Stores and retrieves information across conversations through a per-user memory file directory.'
|
||||
},
|
||||
{
|
||||
title: 'Notepad',
|
||||
link: 'https://github.com/arc53/DocsGPT/blob/main/application/agents/tools/notes.py',
|
||||
link: 'https://github.com/arc53/DocsGPT/blob/main/docsgpt/agents/tools/notes.py',
|
||||
description: 'A single editable note. Supports viewing, overwriting, and string replacement.'
|
||||
},
|
||||
{
|
||||
title: 'Todo List',
|
||||
link: 'https://github.com/arc53/DocsGPT/blob/main/application/agents/tools/todo_list.py',
|
||||
link: 'https://github.com/arc53/DocsGPT/blob/main/docsgpt/agents/tools/todo_list.py',
|
||||
description: 'Manages todo items — creating, viewing, updating, and deleting todos.'
|
||||
}
|
||||
]}
|
||||
|
||||
@@ -27,7 +27,7 @@ While DocsGPT offers a range of built-in tools and a versatile API Tool, there a
|
||||
Before you begin, ensure you have:
|
||||
|
||||
* A solid understanding of Python programming.
|
||||
* Familiarity with the DocsGPT project structure, particularly the `application/agents/tools/` directory where custom tools reside.
|
||||
* Familiarity with the DocsGPT project structure, particularly the `docsgpt/agents/tools/` directory where custom tools reside.
|
||||
* Basic knowledge of how APIs work, as many tools involve interacting with external or internal APIs.
|
||||
* Your DocsGPT development environment set up. If not, please refer to the [Setting Up a Development Environment](/Deploying/Development-Environment) guide.
|
||||
|
||||
@@ -35,7 +35,7 @@ Before you begin, ensure you have:
|
||||
|
||||
Custom tools in DocsGPT are Python classes that inherit from a base `Tool` class and implement specific methods to define their behavior, capabilities, and configuration needs.
|
||||
|
||||
The **foundation** for all custom tools is the abstract base class, located in `application/agents/tools/base.py`. Your custom tool class **must** inherit from this class.
|
||||
The **foundation** for all custom tools is the abstract base class, located in `docsgpt/agents/tools/base.py`. Your custom tool class **must** inherit from this class.
|
||||
|
||||
### Essential Methods to Implement
|
||||
|
||||
@@ -150,11 +150,11 @@ Your custom tool class needs to implement the following methods:
|
||||
|
||||
## Tool Registration and Discovery
|
||||
|
||||
DocsGPT's ToolManager (located in application/agents/tools/tool_manager.py) automatically discovers and loads tools.
|
||||
DocsGPT's ToolManager (located in docsgpt/agents/tools/tool_manager.py) automatically discovers and loads tools.
|
||||
|
||||
As long as your custom tool:
|
||||
|
||||
1. Is placed in a Python file within the `application/agents/tools/` directory (and the filename is not `base.py` or starts with `__`).
|
||||
1. Is placed in a Python file within the `docsgpt/agents/tools/` directory (and the filename is not `base.py` or starts with `__`).
|
||||
2. Correctly inherits from the `Tool` base class.
|
||||
3. Implements all the abstract methods (`execute_action`, `get_actions_metadata`, `get_config_requirements`).
|
||||
|
||||
|
||||
@@ -19,8 +19,8 @@ DocsGPT now runs embeddings through [FastEmbed](https://github.com/qdrant/fastem
|
||||
**Your worker command does need one change.** Query embedding now runs on the Celery worker (`EMBEDDINGS_DELEGATE_TO_WORKER`, on by default), which keeps the API from loading a model of its own. If you start your worker with an explicit `-Q`, add the `embeddings` queue:
|
||||
|
||||
```diff
|
||||
- celery -A application.app.celery worker -l INFO -Q docsgpt,parsing
|
||||
+ celery -A application.app.celery worker -l INFO -Q docsgpt,parsing,embeddings
|
||||
- celery -A docsgpt.app.celery worker -l INFO -Q docsgpt,parsing
|
||||
+ celery -A docsgpt.app.celery worker -l INFO -Q docsgpt,parsing,embeddings
|
||||
```
|
||||
|
||||
The bundled Compose and Kubernetes manifests already do this — pull them along with the code. Without it, every search blocks for `EMBEDDINGS_DELEGATE_TIMEOUT` (60s) and then answers with no retrieved context rather than raising, so the symptom is bad answers, not an error. To keep the model out of the worker too, set `EMBEDDINGS_BASE_URL`; to run the API on its own, set `EMBEDDINGS_DELEGATE_TO_WORKER=false`.
|
||||
@@ -41,8 +41,8 @@ Set the model, then rebuild the vectors:
|
||||
EMBEDDINGS_NAME=ibm-granite/granite-embedding-311m-multilingual-r2
|
||||
|
||||
# 2. Rebuild the vectors from the chunk text already in your index
|
||||
docker compose exec backend python -m application.scripts.reembed --dry-run
|
||||
docker compose exec backend python -m application.scripts.reembed
|
||||
docker compose exec backend python -m docsgpt.scripts.reembed --dry-run
|
||||
docker compose exec backend python -m docsgpt.scripts.reembed
|
||||
```
|
||||
|
||||
Re-embedding reads the chunk text already stored in your index. It does not re-download, re-parse or re-chunk your documents, so no source files are needed and the run is proportional to index size, not corpus size. Both `pgvector` and `faiss` are supported.
|
||||
@@ -90,10 +90,32 @@ need attention:
|
||||
Staying on `all-mpnet-base-v2` is a supported choice — it remains in the model registry and in `setup.sh`. You only need this section if you want to move to granite.
|
||||
</Callout>
|
||||
|
||||
## Backend package renamed to `docsgpt`
|
||||
|
||||
The backend's Python package is `docsgpt` (it was `application`), the name it
|
||||
will carry on PyPI. For one release the old name keeps working through an
|
||||
alias, so nothing breaks on upgrade, but update these before the alias goes:
|
||||
|
||||
- Entry points: `celery -A docsgpt.app.celery worker`,
|
||||
`uvicorn docsgpt.asgi:asgi_app`, `python -m docsgpt.scripts.<name>`. The
|
||||
`application.…` spellings still run and print a `FutureWarning`. The
|
||||
compose files, Kubernetes manifests and setup scripts in the repository are
|
||||
already updated; only custom copies need editing.
|
||||
- Local image builds: the build context is the repository root, so use
|
||||
`docker build -f docsgpt/Dockerfile .` (or the compose files, which do this).
|
||||
- Celery task names changed with the package (`docsgpt.api.user.tasks.ingest`
|
||||
and so on). A worker on this release also accepts the old names, so tasks
|
||||
queued before the upgrade still run, and beat rewrites the periodic
|
||||
schedule in Redis on start-up. The daily, weekly and monthly source-sync
|
||||
timers restart from the upgrade, so the first sync after it can land later
|
||||
than it would have (a monthly sync by up to a month). Nothing to do.
|
||||
- Data directories do not move: the compose files keep your indexes, inputs
|
||||
and vectors under `application/` in the checkout, where they already are.
|
||||
|
||||
## Check your version
|
||||
|
||||
```bash
|
||||
docker compose exec backend python -c "from application.version import get_version; print(get_version())"
|
||||
docker compose exec backend python -c "from docsgpt.version import get_version; print(get_version())"
|
||||
```
|
||||
|
||||
Release notes: [changelog](/changelog). Tags: [GitHub releases](https://github.com/arc53/DocsGPT/releases).
|
||||
@@ -135,7 +157,7 @@ Full manifests: [Kubernetes deployment guide](/Deploying/Kubernetes-Deploying).
|
||||
Alembic migrations run on worker startup. To apply manually:
|
||||
|
||||
```bash
|
||||
docker compose exec backend alembic -c application/alembic.ini upgrade head
|
||||
docker compose exec backend alembic -c docsgpt/alembic.ini upgrade head
|
||||
```
|
||||
|
||||
`upgrade head` is idempotent.
|
||||
|
||||
@@ -126,7 +126,7 @@ reconnect HTTP errored. Common cases:
|
||||
- The user's JWT rotated mid-stream → 401 on the GET. Frontend
|
||||
doesn't auto-refresh; the user reloads.
|
||||
- The user is on a different host than the API and CORS is rejecting
|
||||
the GET → check `application/asgi.py` allow-headers.
|
||||
the GET → check `docsgpt/asgi.py` allow-headers.
|
||||
|
||||
### D. "The dev install never delivers any notifications at all"
|
||||
|
||||
@@ -316,7 +316,7 @@ redis-cli -n 2 DEL user:<id>:stream
|
||||
|
||||
## Settings reference
|
||||
|
||||
Everything in `application/core/settings.py`:
|
||||
Everything in `docsgpt/core/settings.py`:
|
||||
|
||||
| Setting | Default | Purpose |
|
||||
| --------------------------------------------- | ------- | --------------------------------------------- |
|
||||
@@ -364,16 +364,16 @@ re-surface UX is too aggressive for v2.
|
||||
|
||||
The chat-stream reconnect reader `GET /api/messages/<id>/events`
|
||||
is a native-async Starlette route mounted in
|
||||
`application/asgi.py`, not a Flask route. Plain `flask run`
|
||||
`docsgpt/asgi.py`, not a Flask route. Plain `flask run`
|
||||
serves only the WSGI Flask app, so under it that endpoint 404s
|
||||
and reconnect-after-disconnect can't resume. Run the backend via
|
||||
`uvicorn application.asgi:asgi_app --reload` (or the production
|
||||
`uvicorn docsgpt.asgi:asgi_app --reload` (or the production
|
||||
gunicorn uvicorn-worker) to exercise it.
|
||||
|
||||
### Werkzeug doesn't auto-reload route files
|
||||
|
||||
The dev server (`flask run`) doesn't watch
|
||||
`application/api/events/routes.py` for changes by default.
|
||||
`docsgpt/api/events/routes.py` for changes by default.
|
||||
After editing the route, restart Flask manually — `--reload`
|
||||
isn't on. (Production gunicorn reloads via deploy.)
|
||||
|
||||
|
||||
@@ -0,0 +1,175 @@
|
||||
# DocsGPT backend image.
|
||||
#
|
||||
# Build args:
|
||||
# EXTRAS comma-separated optional extras to bake in, matching the
|
||||
# pyproject extras / requirements-<extra>.txt files:
|
||||
# docling (layout-model parser + OCR backend), milvus.
|
||||
# INSTALL_DOCLING legacy alias for EXTRAS=docling (setup.sh writes it).
|
||||
# INSTALL_TESSERACT bake the tesseract binary for OCR_ENGINE=tesseract.
|
||||
# EMBEDDINGS_PREFETCH registry names of the embedding models to bake; empty
|
||||
# bakes both defaults (mpnet for upgrades, granite for
|
||||
# new installs).
|
||||
#
|
||||
# Everything the default configuration needs is inside the image: embedding
|
||||
# models, their tokenizers, tiktoken's encoding and, with the docling extra,
|
||||
# docling's layout/table/OCR models. `python -m docsgpt.scripts.verify_offline`
|
||||
# under `docker run --network none` proves it.
|
||||
|
||||
FROM ubuntu:24.04 AS builder
|
||||
|
||||
ENV DEBIAN_FRONTEND=noninteractive
|
||||
|
||||
# Ubuntu 24.04 ships Python 3.12 in its main archive: no PPA needed. Every pin
|
||||
# resolves to a wheel, so no compiler toolchain either.
|
||||
RUN apt-get update && \
|
||||
apt-get install -y --no-install-recommends python3.12 python3.12-venv ca-certificates && \
|
||||
rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# Build context is the repository root (see .dockerignore there):
|
||||
# docker build -f docsgpt/Dockerfile .
|
||||
COPY docsgpt/requirements.txt docsgpt/requirements-docling.txt docsgpt/requirements-milvus.txt ./
|
||||
|
||||
RUN python3.12 -m venv /venv
|
||||
ENV PATH="/venv/bin:$PATH"
|
||||
|
||||
RUN pip install --no-cache-dir --upgrade pip && \
|
||||
pip install --no-cache-dir --only-binary=:all: -r requirements.txt
|
||||
|
||||
# Optional extras. Each requirements-<extra>.txt is exported from the same
|
||||
# lock as requirements.txt, so installing it on top only adds the extra's
|
||||
# packages. The docling file takes torch from the CPU-only PyTorch index.
|
||||
# Not wheels-only: docling's antlr4 runtime ships as a pure-Python sdist.
|
||||
ARG EXTRAS=""
|
||||
ARG INSTALL_DOCLING=false
|
||||
RUN set -e; \
|
||||
extras="$EXTRAS"; \
|
||||
if [ "$INSTALL_DOCLING" = "true" ]; then extras="$extras,docling"; fi; \
|
||||
for extra in $(echo "$extras" | tr ',' ' '); do \
|
||||
echo "Installing extra: $extra"; \
|
||||
pip install --no-cache-dir -r "requirements-$extra.txt"; \
|
||||
done
|
||||
|
||||
# google-api-python-client bundles discovery documents for ~600 Google APIs
|
||||
# (99 MB). The application builds one client, Drive v3; keep only its document.
|
||||
# Building another API's client needs its file back, or static_discovery=False.
|
||||
RUN find /venv/lib/python3.12/site-packages/googleapiclient/discovery_cache/documents \
|
||||
-type f ! -name 'drive.v3.json' -delete
|
||||
|
||||
|
||||
FROM ubuntu:24.04 AS final
|
||||
|
||||
ENV DEBIAN_FRONTEND=noninteractive
|
||||
|
||||
RUN apt-get update && \
|
||||
apt-get install -y --no-install-recommends python3.12 poppler-utils ca-certificates && \
|
||||
ln -s /usr/bin/python3.12 /usr/bin/python && \
|
||||
rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# opencv (rapidocr, part of the docling extra) needs libGL at import time.
|
||||
ARG EXTRAS=""
|
||||
ARG INSTALL_DOCLING=false
|
||||
RUN if [ "$INSTALL_DOCLING" = "true" ] || echo ",$EXTRAS," | grep -q ",docling,"; then \
|
||||
apt-get update && \
|
||||
apt-get install -y --no-install-recommends libgl1 libglib2.0-0 && \
|
||||
rm -rf /var/lib/apt/lists/*; \
|
||||
fi
|
||||
|
||||
# Optional tesseract OCR engine (OCR_ENABLED=true with OCR_ENGINE=tesseract,
|
||||
# the default engine); ~35 MB of system packages. Extra language packs are a
|
||||
# deployment concern (apt: tesseract-ocr-<lang>, then list them in OCR_LANGS).
|
||||
# A DeepSeek-OCR endpoint (OCR_ENGINE=deepseek) needs none of this.
|
||||
ARG INSTALL_TESSERACT=false
|
||||
RUN if [ "$INSTALL_TESSERACT" = "true" ]; then \
|
||||
apt-get update && \
|
||||
apt-get install -y --no-install-recommends tesseract-ocr tesseract-ocr-eng && \
|
||||
rm -rf /var/lib/apt/lists/*; \
|
||||
fi
|
||||
|
||||
LABEL org.opencontainers.image.source="https://github.com/arc53/DocsGPT" \
|
||||
org.opencontainers.image.title="DocsGPT" \
|
||||
org.opencontainers.image.description="DocsGPT backend: API and Celery worker" \
|
||||
org.opencontainers.image.licenses="MIT"
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
# The process user owns /app so the model prefetch below can run as it: an
|
||||
# unprivileged prefetch writes the model files with the right owner up front,
|
||||
# instead of a trailing chown -R that rewrites every model file into a second
|
||||
# layer.
|
||||
RUN groupadd -r appuser && \
|
||||
useradd -r -g appuser -d /app -s /sbin/nologin -c "Docker image user" appuser && \
|
||||
chown appuser:appuser /app && \
|
||||
install -d -o appuser -g appuser /app/models /app/docsgpt
|
||||
|
||||
COPY --from=builder /venv /venv
|
||||
|
||||
# Every cache the application reads at run time lives under /app/models and is
|
||||
# filled at build time:
|
||||
# EMBEDDINGS_CACHE_DIR / HF_HUB_CACHE FastEmbed models and their tokenizers
|
||||
# (chunking reads tokenizer.json from
|
||||
# the same hub-layout snapshot)
|
||||
# TIKTOKEN_CACHE_DIR cl100k_base for token accounting
|
||||
# DOCLING_ARTIFACTS_PATH docling's models (docling extra only)
|
||||
ENV EMBEDDINGS_CACHE_DIR=/app/models \
|
||||
HF_HUB_CACHE=/app/models \
|
||||
TIKTOKEN_CACHE_DIR=/app/models/tiktoken \
|
||||
DOCLING_ARTIFACTS_PATH=/app/models/docling \
|
||||
HF_HUB_DISABLE_TELEMETRY=1 \
|
||||
PATH="/venv/bin:$PATH"
|
||||
|
||||
# Only the modules the prefetch imports are copied first, so an unrelated
|
||||
# source edit does not invalidate the model layer.
|
||||
COPY --chown=appuser:appuser docsgpt/__init__.py /app/docsgpt/__init__.py
|
||||
COPY --chown=appuser:appuser docsgpt/scripts/__init__.py docsgpt/scripts/prefetch_models.py /app/docsgpt/scripts/
|
||||
COPY --chown=appuser:appuser docsgpt/vectorstore/__init__.py docsgpt/vectorstore/model_registry.py /app/docsgpt/vectorstore/
|
||||
|
||||
USER appuser
|
||||
|
||||
ARG EMBEDDINGS_PREFETCH=""
|
||||
RUN PYTHONPATH=/app python -m docsgpt.scripts.prefetch_models ${EMBEDDINGS_PREFETCH} && \
|
||||
rm -rf /app/models/.locks /app/.cache
|
||||
|
||||
# docling downloads its layout, table-structure and OCR models on first parse;
|
||||
# bake them so the docling variant is as self-contained as the default image.
|
||||
RUN if python -c "import docling" 2>/dev/null; then \
|
||||
docling-tools models download --output-dir /app/models/docling layout tableformer rapidocr && \
|
||||
rm -rf /app/.cache; \
|
||||
fi
|
||||
|
||||
COPY --chown=appuser:appuser docsgpt /app/docsgpt
|
||||
# One-release alias so `-A application.app.celery` style entry points keep working.
|
||||
COPY --chown=appuser:appuser application/__init__.py /app/application/__init__.py
|
||||
|
||||
# Runtime data directories, owned by the process user so a named volume
|
||||
# mounted on them (docker-compose-standalone.yaml) inherits that ownership
|
||||
# and uploads work without running the container as root.
|
||||
RUN mkdir -p /app/docsgpt/inputs/local /app/inputs /app/indexes /app/vectors
|
||||
|
||||
ENV FLASK_APP=app.py
|
||||
|
||||
# Thread caps. onnxruntime (FastEmbed) ignores OMP_NUM_THREADS and sizes its
|
||||
# pool to the host's core count, which a CPU-limited container still reports;
|
||||
# EMBEDDINGS_THREADS pins it the way OMP_NUM_THREADS pinned torch before.
|
||||
ENV MALLOC_ARENA_MAX=2 \
|
||||
OMP_NUM_THREADS=4 \
|
||||
MKL_NUM_THREADS=4 \
|
||||
OPENBLAS_NUM_THREADS=4 \
|
||||
EMBEDDINGS_THREADS=4
|
||||
|
||||
EXPOSE 7091
|
||||
|
||||
# BoundedDrainUvicornWorker makes max_requests recycles safe with held-open SSE
|
||||
# connections (see docsgpt/gunicorn_worker.py); with recycles now safe,
|
||||
# --max-requests is raised (kept for memory hygiene) to cut churn.
|
||||
CMD ["gunicorn", \
|
||||
"-w", "1", \
|
||||
"-k", "docsgpt.gunicorn_worker.BoundedDrainUvicornWorker", \
|
||||
"--bind", "0.0.0.0:7091", \
|
||||
"--timeout", "180", \
|
||||
"--graceful-timeout", "120", \
|
||||
"--keep-alive", "5", \
|
||||
"--worker-tmp-dir", "/dev/shm", \
|
||||
"--max-requests", "5000", \
|
||||
"--max-requests-jitter", "500", \
|
||||
"--config", "docsgpt/gunicorn_conf.py", \
|
||||
"docsgpt.asgi:asgi_app"]
|
||||
File renamed without changes.
File renamed without changes.
@@ -1,9 +1,9 @@
|
||||
import logging
|
||||
|
||||
from application.agents.agentic_agent import AgenticAgent
|
||||
from application.agents.classic_agent import ClassicAgent
|
||||
from application.agents.research_agent import ResearchAgent
|
||||
from application.agents.workflow_agent import WorkflowAgent
|
||||
from docsgpt.agents.agentic_agent import AgenticAgent
|
||||
from docsgpt.agents.classic_agent import ClassicAgent
|
||||
from docsgpt.agents.research_agent import ResearchAgent
|
||||
from docsgpt.agents.workflow_agent import WorkflowAgent
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -1,10 +1,10 @@
|
||||
import logging
|
||||
from typing import Dict, Generator, Optional
|
||||
|
||||
from application.agents.base import BaseAgent
|
||||
from application.agents.tools.internal_search import add_internal_search_tool
|
||||
from application.agents.tools.wiki import add_wiki_tool
|
||||
from application.logging import LogContext
|
||||
from docsgpt.agents.base import BaseAgent
|
||||
from docsgpt.agents.tools.internal_search import add_internal_search_tool
|
||||
from docsgpt.agents.tools.wiki import add_wiki_tool
|
||||
from docsgpt.logging import LogContext
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -6,30 +6,30 @@ from abc import ABC, abstractmethod
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any, Dict, Generator, List, Optional
|
||||
|
||||
from application.agents.tool_executor import (
|
||||
from docsgpt.agents.tool_executor import (
|
||||
ToolExecutor,
|
||||
result_status,
|
||||
truncate_tool_result,
|
||||
)
|
||||
from application.core.json_schema_utils import (
|
||||
from docsgpt.core.json_schema_utils import (
|
||||
JsonSchemaValidationError,
|
||||
normalize_json_schema_payload,
|
||||
)
|
||||
from application.core.settings import settings
|
||||
from application.llm.handlers.base import (
|
||||
from docsgpt.core.settings import settings
|
||||
from docsgpt.llm.handlers.base import (
|
||||
ToolCall,
|
||||
_bound_tool_response_for_llm,
|
||||
)
|
||||
from application.guardrails.config import DEFAULT_BLOCK_MESSAGE as GUARDRAIL_DEFAULT_MESSAGE
|
||||
from application.guardrails.runtime import (
|
||||
from docsgpt.guardrails.config import DEFAULT_BLOCK_MESSAGE as GUARDRAIL_DEFAULT_MESSAGE
|
||||
from docsgpt.guardrails.runtime import (
|
||||
build_engine as build_guardrail_engine,
|
||||
resolve_config as resolve_guardrails_config,
|
||||
)
|
||||
from application.guardrails.stream import StreamingOutputGuard
|
||||
from application.guardrails.types import Action, Stage, resolve_tool_result
|
||||
from application.llm.handlers.handler_creator import LLMHandlerCreator
|
||||
from application.llm.llm_creator import LLMCreator
|
||||
from application.logging import build_stack_data, log_activity, LogContext
|
||||
from docsgpt.guardrails.stream import StreamingOutputGuard
|
||||
from docsgpt.guardrails.types import Action, Stage, resolve_tool_result
|
||||
from docsgpt.llm.handlers.handler_creator import LLMHandlerCreator
|
||||
from docsgpt.llm.llm_creator import LLMCreator
|
||||
from docsgpt.logging import build_stack_data, log_activity, LogContext
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -424,7 +424,7 @@ class BaseAgent(ABC):
|
||||
return meta["response_id"]
|
||||
budget = getattr(settings, "OPENAI_RESPONSES_CHAIN_BUDGET_TOKENS", None)
|
||||
if not budget:
|
||||
from application.core.model_utils import get_token_limit
|
||||
from docsgpt.core.model_utils import get_token_limit
|
||||
|
||||
budget = get_token_limit(
|
||||
getattr(self, "model_id", None),
|
||||
@@ -677,13 +677,13 @@ class BaseAgent(ABC):
|
||||
# ---- Context / token management ----
|
||||
|
||||
def _calculate_current_context_tokens(self, messages: List[Dict]) -> int:
|
||||
from application.api.answer.services.compression.token_counter import (
|
||||
from docsgpt.api.answer.services.compression.token_counter import (
|
||||
TokenCounter,
|
||||
)
|
||||
return TokenCounter.count_message_tokens(messages)
|
||||
|
||||
def _check_context_limit(self, messages: List[Dict]) -> bool:
|
||||
from application.core.model_utils import get_token_limit
|
||||
from docsgpt.core.model_utils import get_token_limit
|
||||
|
||||
try:
|
||||
current_tokens = self._calculate_current_context_tokens(messages)
|
||||
@@ -705,7 +705,7 @@ class BaseAgent(ABC):
|
||||
return False
|
||||
|
||||
def _validate_context_size(self, messages: List[Dict]) -> None:
|
||||
from application.core.model_utils import get_token_limit
|
||||
from docsgpt.core.model_utils import get_token_limit
|
||||
|
||||
current_tokens = self._calculate_current_context_tokens(messages)
|
||||
self.current_token_count = current_tokens
|
||||
@@ -728,7 +728,7 @@ class BaseAgent(ABC):
|
||||
)
|
||||
|
||||
def _truncate_text_middle(self, text: str, max_tokens: int) -> str:
|
||||
from application.utils import num_tokens_from_string
|
||||
from docsgpt.utils import num_tokens_from_string
|
||||
|
||||
current_tokens = num_tokens_from_string(text)
|
||||
if current_tokens <= max_tokens:
|
||||
@@ -767,8 +767,8 @@ class BaseAgent(ABC):
|
||||
(the usual culprit) and raises when even that cannot fit — BEFORE
|
||||
the usage decorators run, so a hopeless payload costs nothing.
|
||||
"""
|
||||
from application.core.model_utils import get_token_limit
|
||||
from application.utils import num_tokens_from_string
|
||||
from docsgpt.core.model_utils import get_token_limit
|
||||
from docsgpt.utils import num_tokens_from_string
|
||||
|
||||
context_limit = get_token_limit(
|
||||
self.model_id, user_id=self.model_user_id or self.user
|
||||
@@ -847,7 +847,7 @@ class BaseAgent(ABC):
|
||||
"""
|
||||
if getattr(self, "prompt_embeds_documents", False):
|
||||
return ""
|
||||
from application.api.answer.services.prompt_renderer import (
|
||||
from docsgpt.api.answer.services.prompt_renderer import (
|
||||
format_docs_for_prompt,
|
||||
)
|
||||
|
||||
@@ -887,7 +887,7 @@ class BaseAgent(ABC):
|
||||
attacker-influenceable material. The prompt was rendered before the
|
||||
agent ran, so the verdict is applied by patching it here.
|
||||
"""
|
||||
from application.api.answer.services.prompt_renderer import (
|
||||
from docsgpt.api.answer.services.prompt_renderer import (
|
||||
format_docs_for_prompt,
|
||||
)
|
||||
|
||||
@@ -959,7 +959,7 @@ class BaseAgent(ABC):
|
||||
"""Merge the cached InternalSearchTool's docs into ``retrieved_docs``,
|
||||
deduped, preserving any pre-fetched docs so a mixed-exposure agent cites
|
||||
both pre-fetched and tool-retrieved sources (not just the tool's)."""
|
||||
from application.agents.tools.internal_search import INTERNAL_TOOL_ID
|
||||
from docsgpt.agents.tools.internal_search import INTERNAL_TOOL_ID
|
||||
|
||||
executor = getattr(self, "tool_executor", None)
|
||||
loaded = getattr(executor, "_loaded_tools", None) or {}
|
||||
@@ -1004,8 +1004,8 @@ class BaseAgent(ABC):
|
||||
query: str,
|
||||
) -> List[Dict]:
|
||||
"""Build messages using pre-rendered system prompt"""
|
||||
from application.core.model_utils import get_token_limit
|
||||
from application.utils import num_tokens_from_string
|
||||
from docsgpt.core.model_utils import get_token_limit
|
||||
from docsgpt.utils import num_tokens_from_string
|
||||
|
||||
# Retrieval controls run inside _build_document_block for the usual
|
||||
# path; a prompt that embeds the documents skips that block entirely,
|
||||
@@ -1197,7 +1197,7 @@ class BaseAgent(ABC):
|
||||
history: List[Dict],
|
||||
max_tokens: int,
|
||||
) -> List[Dict]:
|
||||
from application.utils import num_tokens_from_string
|
||||
from docsgpt.utils import num_tokens_from_string
|
||||
|
||||
if not history or max_tokens <= 0:
|
||||
return []
|
||||
@@ -1,10 +1,10 @@
|
||||
import logging
|
||||
from typing import Dict, Generator, Optional
|
||||
|
||||
from application.agents.base import BaseAgent
|
||||
from application.agents.tools.internal_search import add_internal_search_tool
|
||||
from application.agents.tools.wiki import add_wiki_tool
|
||||
from application.logging import LogContext
|
||||
from docsgpt.agents.base import BaseAgent
|
||||
from docsgpt.agents.tools.internal_search import add_internal_search_tool
|
||||
from docsgpt.agents.tools.wiki import add_wiki_tool
|
||||
from docsgpt.logging import LogContext
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -8,7 +8,7 @@ import logging
|
||||
import uuid
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
from application.core.settings import settings
|
||||
from docsgpt.core.settings import settings
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -61,15 +61,15 @@ _builtin_loaded_cache: Dict[tuple, List[str]] = {}
|
||||
def _load_tool(tool_name: str) -> Optional[Any]:
|
||||
"""Return a metadata-only instance of a tool, or None if it has no class."""
|
||||
# Imports just the named module (not the whole package) — avoids the
|
||||
# circular import via ``mcp_tool`` → ``application.api.user``.
|
||||
# circular import via ``mcp_tool`` → ``docsgpt.api.user``.
|
||||
if tool_name in _tool_cache:
|
||||
return _tool_cache[tool_name]
|
||||
|
||||
from application.agents.tools.base import Tool
|
||||
from docsgpt.agents.tools.base import Tool
|
||||
|
||||
instance: Optional[Any] = None
|
||||
try:
|
||||
module = importlib.import_module(f"application.agents.tools.{tool_name}")
|
||||
module = importlib.import_module(f"docsgpt.agents.tools.{tool_name}")
|
||||
except ModuleNotFoundError:
|
||||
_tool_cache[tool_name] = None
|
||||
return None
|
||||
@@ -5,19 +5,19 @@ from __future__ import annotations
|
||||
import logging
|
||||
from typing import Any, Dict, Iterable, List, Optional
|
||||
|
||||
from application.agents.agent_creator import AgentCreator
|
||||
from application.agents.tool_executor import ToolExecutor
|
||||
from application.api.answer.services.prompt_renderer import (
|
||||
from docsgpt.agents.agent_creator import AgentCreator
|
||||
from docsgpt.agents.tool_executor import ToolExecutor
|
||||
from docsgpt.api.answer.services.prompt_renderer import (
|
||||
PromptRenderer,
|
||||
format_docs_for_prompt,
|
||||
prompt_embeds_documents,
|
||||
resolve_prompt_skeleton,
|
||||
)
|
||||
from application.api.answer.services.stream_processor import get_prompt
|
||||
from application.core.settings import settings
|
||||
from application.retriever.retriever_creator import RetrieverCreator
|
||||
from application.storage.db.repositories.sources import SourcesRepository
|
||||
from application.storage.db.session import db_readonly
|
||||
from docsgpt.api.answer.services.stream_processor import get_prompt
|
||||
from docsgpt.core.settings import settings
|
||||
from docsgpt.retriever.retriever_creator import RetrieverCreator
|
||||
from docsgpt.storage.db.repositories.sources import SourcesRepository
|
||||
from docsgpt.storage.db.session import db_readonly
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -70,13 +70,13 @@ def run_agent_headless(
|
||||
conversation_id: Optional[str] = None,
|
||||
) -> Dict[str, Any]:
|
||||
"""Run an agent with no live client; returns a structured outcome dict."""
|
||||
from application.core.model_utils import (
|
||||
from docsgpt.core.model_utils import (
|
||||
get_api_key_for_provider,
|
||||
get_default_model_id,
|
||||
get_provider_from_model_id,
|
||||
validate_model_id,
|
||||
)
|
||||
from application.utils import calculate_doc_token_budget
|
||||
from docsgpt.utils import calculate_doc_token_budget
|
||||
|
||||
owner = _resolve_owner(agent_config)
|
||||
if not owner:
|
||||
@@ -210,7 +210,7 @@ def run_agent_headless(
|
||||
# by raising. Dropping it here (as this loop used to) makes a broken run
|
||||
# indistinguishable from one that simply had nothing to say, and the
|
||||
# caller records it as a success. Mirrors the sentinel in
|
||||
# ``application/logging.py`` so an error carrying no message is still
|
||||
# ``docsgpt/logging.py`` so an error carrying no message is still
|
||||
# truthy instead of reading as "ok".
|
||||
if event.get("type") == "error":
|
||||
stream_error = str(event.get("error") or "")[:500] or "unspecified"
|
||||
@@ -4,15 +4,15 @@ import os
|
||||
import time
|
||||
from typing import Dict, Generator, List, Optional
|
||||
|
||||
from application.agents.base import BaseAgent
|
||||
from application.agents.tool_executor import ToolExecutor
|
||||
from application.agents.tools.internal_search import (
|
||||
from docsgpt.agents.base import BaseAgent
|
||||
from docsgpt.agents.tool_executor import ToolExecutor
|
||||
from docsgpt.agents.tools.internal_search import (
|
||||
INTERNAL_TOOL_ID,
|
||||
add_internal_search_tool,
|
||||
)
|
||||
from application.agents.tools.wiki import add_wiki_tool
|
||||
from application.agents.tools.think import THINK_TOOL_ENTRY, THINK_TOOL_ID
|
||||
from application.logging import LogContext
|
||||
from docsgpt.agents.tools.wiki import add_wiki_tool
|
||||
from docsgpt.agents.tools.think import THINK_TOOL_ENTRY, THINK_TOOL_ID
|
||||
from docsgpt.logging import LogContext
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -673,7 +673,7 @@ class ResearchAgent(BaseAgent):
|
||||
)
|
||||
|
||||
if log_context:
|
||||
from application.logging import build_stack_data
|
||||
from docsgpt.logging import build_stack_data
|
||||
|
||||
log_context.stacks.append(
|
||||
{"component": "synthesis_llm", "data": build_stack_data(self.llm)}
|
||||
File renamed without changes.
@@ -6,25 +6,25 @@ from typing import Any, Dict, List, Optional, Tuple
|
||||
|
||||
from sqlalchemy.exc import IntegrityError
|
||||
|
||||
from application.agents.default_tools import (
|
||||
from docsgpt.agents.default_tools import (
|
||||
BUILTIN_AGENT_TOOLS,
|
||||
is_headless_excluded_tool,
|
||||
is_synthesized_tool_id,
|
||||
resolve_tool_by_id,
|
||||
synthesized_default_tools,
|
||||
)
|
||||
from application.agents.tools.tool_action_parser import ToolActionParser
|
||||
from application.agents.tools.tool_manager import ToolManager
|
||||
from application.guardrails.types import Stage as GuardrailStage, resolve_tool_result
|
||||
from application.security.encryption import decrypt_credentials
|
||||
from application.storage.db.base_repository import looks_like_uuid
|
||||
from application.storage.db.repositories.agents import AgentsRepository
|
||||
from application.storage.db.repositories.tool_call_attempts import (
|
||||
from docsgpt.agents.tools.tool_action_parser import ToolActionParser
|
||||
from docsgpt.agents.tools.tool_manager import ToolManager
|
||||
from docsgpt.guardrails.types import Stage as GuardrailStage, resolve_tool_result
|
||||
from docsgpt.security.encryption import decrypt_credentials
|
||||
from docsgpt.storage.db.base_repository import looks_like_uuid
|
||||
from docsgpt.storage.db.repositories.agents import AgentsRepository
|
||||
from docsgpt.storage.db.repositories.tool_call_attempts import (
|
||||
ToolCallAttemptsRepository,
|
||||
)
|
||||
from application.storage.db.repositories.user_tools import UserToolsRepository
|
||||
from application.storage.db.repositories.users import UsersRepository
|
||||
from application.storage.db.session import db_readonly, db_session
|
||||
from docsgpt.storage.db.repositories.user_tools import UserToolsRepository
|
||||
from docsgpt.storage.db.repositories.users import UsersRepository
|
||||
from docsgpt.storage.db.session import db_readonly, db_session
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -51,7 +51,7 @@ def _dedupable_tool_names() -> frozenset:
|
||||
Only these are safe to collapse — an MCP or user-added tool may
|
||||
legitimately appear more than once under a single name.
|
||||
"""
|
||||
from application.core.settings import settings
|
||||
from docsgpt.core.settings import settings
|
||||
|
||||
return frozenset(BUILTIN_AGENT_TOOLS) | frozenset(getattr(settings, "DEFAULT_CHAT_TOOLS", None) or [])
|
||||
|
||||
@@ -846,7 +846,7 @@ class ToolExecutor:
|
||||
silently bypasses the prompt — not even via the headless allowlist.
|
||||
"""
|
||||
try:
|
||||
from application.agents.tools.remote_device import RemoteDeviceTool
|
||||
from docsgpt.agents.tools.remote_device import RemoteDeviceTool
|
||||
|
||||
tool = RemoteDeviceTool(
|
||||
config=tool_data.get("config") or {},
|
||||
@@ -872,7 +872,7 @@ class ToolExecutor:
|
||||
error so a misconfigured tool never silently runs untrusted code.
|
||||
"""
|
||||
try:
|
||||
from application.agents.tools.code_executor import CodeExecutorTool
|
||||
from docsgpt.agents.tools.code_executor import CodeExecutorTool
|
||||
|
||||
tool = CodeExecutorTool(
|
||||
tool_config=tool_data.get("config") or {},
|
||||
File renamed without changes.
@@ -6,12 +6,12 @@ from urllib.parse import quote, urlencode
|
||||
|
||||
import requests
|
||||
|
||||
from application.agents.tools.api_body_serializer import (
|
||||
from docsgpt.agents.tools.api_body_serializer import (
|
||||
ContentType,
|
||||
RequestBodySerializer,
|
||||
)
|
||||
from application.agents.tools.base import Tool
|
||||
from application.security.safe_url import UnsafeUserUrlError, pinned_request
|
||||
from docsgpt.agents.tools.base import Tool
|
||||
from docsgpt.security.safe_url import UnsafeUserUrlError, pinned_request
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
+7
-7
@@ -17,17 +17,17 @@ import logging
|
||||
import uuid
|
||||
from typing import Any, Dict, List, Optional, Tuple
|
||||
|
||||
from application.agents.tools.artifact_ref import resolve_artifact_id
|
||||
from application.agents.tools.base import Tool
|
||||
from application.core.settings import settings
|
||||
from application.sandbox.artifacts_capture import (
|
||||
from docsgpt.agents.tools.artifact_ref import resolve_artifact_id
|
||||
from docsgpt.agents.tools.base import Tool
|
||||
from docsgpt.core.settings import settings
|
||||
from docsgpt.sandbox.artifacts_capture import (
|
||||
QuotaExceeded,
|
||||
append_artifact_version,
|
||||
persist_new_artifact,
|
||||
)
|
||||
from application.sandbox.sandbox_creator import SandboxCreator
|
||||
from application.storage.db.repositories.artifacts import ArtifactsRepository
|
||||
from application.storage.db.session import db_readonly
|
||||
from docsgpt.sandbox.sandbox_creator import SandboxCreator
|
||||
from docsgpt.storage.db.repositories.artifacts import ArtifactsRepository
|
||||
from docsgpt.storage.db.session import db_readonly
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -13,7 +13,7 @@ from __future__ import annotations
|
||||
import re
|
||||
from typing import Any, Optional
|
||||
|
||||
from application.storage.db.base_repository import looks_like_uuid
|
||||
from docsgpt.storage.db.base_repository import looks_like_uuid
|
||||
|
||||
_REF_RE = re.compile(r"^[Aa](\d+)$")
|
||||
|
||||
+6
-6
@@ -12,12 +12,12 @@ from __future__ import annotations
|
||||
import logging
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
from application.core.settings import settings
|
||||
from application.sandbox.artifacts_capture import QuotaExceeded, persist_new_artifact
|
||||
from application.storage.db.repositories.artifacts import ArtifactsRepository
|
||||
from application.storage.db.repositories.attachments import AttachmentsRepository
|
||||
from application.storage.db.session import db_readonly
|
||||
from application.storage.storage_creator import StorageCreator
|
||||
from docsgpt.core.settings import settings
|
||||
from docsgpt.sandbox.artifacts_capture import QuotaExceeded, persist_new_artifact
|
||||
from docsgpt.storage.db.repositories.artifacts import ArtifactsRepository
|
||||
from docsgpt.storage.db.repositories.attachments import AttachmentsRepository
|
||||
from docsgpt.storage.db.session import db_readonly
|
||||
from docsgpt.storage.storage_creator import StorageCreator
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
File renamed without changes.
@@ -2,7 +2,7 @@ import logging
|
||||
|
||||
import requests
|
||||
|
||||
from application.agents.tools.base import Tool
|
||||
from docsgpt.agents.tools.base import Tool
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -6,32 +6,32 @@ import logging
|
||||
import re
|
||||
from typing import Any, Dict, List, Optional, Tuple
|
||||
|
||||
from application.agents.tools.artifact_ref import resolve_artifact_id
|
||||
from application.agents.tools.attachment_bridge import (
|
||||
from docsgpt.agents.tools.artifact_ref import resolve_artifact_id
|
||||
from docsgpt.agents.tools.attachment_bridge import (
|
||||
AttachmentBridgeError,
|
||||
bridge_attachment,
|
||||
match_attachment,
|
||||
)
|
||||
from application.agents.tools.base import Tool
|
||||
from application.core.settings import settings
|
||||
from application.sandbox.artifacts_capture import (
|
||||
from docsgpt.agents.tools.base import Tool
|
||||
from docsgpt.core.settings import settings
|
||||
from docsgpt.sandbox.artifacts_capture import (
|
||||
MAX_CAPTURED_FILES,
|
||||
capture_artifacts,
|
||||
snapshot_signatures,
|
||||
unique_input_path,
|
||||
)
|
||||
from application.sandbox.artifacts_capture import (
|
||||
from docsgpt.sandbox.artifacts_capture import (
|
||||
infer_mime as _infer_mime,
|
||||
)
|
||||
from application.sandbox.artifacts_capture import (
|
||||
from docsgpt.sandbox.artifacts_capture import (
|
||||
kind_for_mime as _kind_for_mime,
|
||||
)
|
||||
from application.sandbox.base import ExecResult
|
||||
from application.sandbox.sandbox_creator import SandboxCreator
|
||||
from application.storage.db.repositories.artifacts import ArtifactsRepository
|
||||
from application.storage.db.session import db_readonly
|
||||
from application.storage.storage_creator import StorageCreator
|
||||
from application.utils import safe_filename
|
||||
from docsgpt.sandbox.base import ExecResult
|
||||
from docsgpt.sandbox.sandbox_creator import SandboxCreator
|
||||
from docsgpt.storage.db.repositories.artifacts import ArtifactsRepository
|
||||
from docsgpt.storage.db.session import db_readonly
|
||||
from docsgpt.storage.storage_creator import StorageCreator
|
||||
from docsgpt.utils import safe_filename
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import requests
|
||||
from application.agents.tools.base import Tool
|
||||
from docsgpt.agents.tools.base import Tool
|
||||
|
||||
|
||||
class CryptoPriceTool(Tool):
|
||||
@@ -2,7 +2,7 @@ import logging
|
||||
import time
|
||||
from typing import Any, Dict, Optional
|
||||
|
||||
from application.agents.tools.base import Tool
|
||||
from docsgpt.agents.tools.base import Tool
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
+7
-7
@@ -2,10 +2,10 @@ import json
|
||||
import logging
|
||||
from typing import Dict, List, Optional
|
||||
|
||||
from application.agents.tools.base import Tool
|
||||
from application.core.settings import settings
|
||||
from application.retriever.dispatcher import build_dispatcher
|
||||
from application.retriever.retriever_creator import RetrieverCreator
|
||||
from docsgpt.agents.tools.base import Tool
|
||||
from docsgpt.core.settings import settings
|
||||
from docsgpt.retriever.dispatcher import build_dispatcher
|
||||
from docsgpt.retriever.retriever_creator import RetrieverCreator
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -78,10 +78,10 @@ class InternalSearchTool(Tool):
|
||||
# Per-operation session: this tool runs inside the answer
|
||||
# generator hot path, so we open a short-lived read
|
||||
# connection for the batch lookup and release immediately.
|
||||
from application.storage.db.repositories.sources import (
|
||||
from docsgpt.storage.db.repositories.sources import (
|
||||
SourcesRepository,
|
||||
)
|
||||
from application.storage.db.session import db_readonly
|
||||
from docsgpt.storage.db.session import db_readonly
|
||||
|
||||
if isinstance(active_docs, str):
|
||||
active_docs = [active_docs]
|
||||
@@ -402,7 +402,7 @@ def sources_have_directory_structure(source: Dict) -> bool:
|
||||
# sites are updated to propagate user context.
|
||||
from sqlalchemy import text as _text
|
||||
|
||||
from application.storage.db.session import db_readonly
|
||||
from docsgpt.storage.db.session import db_readonly
|
||||
|
||||
if isinstance(active_docs, str):
|
||||
active_docs = [active_docs]
|
||||
@@ -19,13 +19,13 @@ from mcp.shared.auth import OAuthClientInformationFull, OAuthClientMetadata, OAu
|
||||
from pydantic import AnyHttpUrl, ValidationError
|
||||
from redis import Redis
|
||||
|
||||
from application.agents.tools.base import Tool
|
||||
from application.api.user.tasks import mcp_oauth_task
|
||||
from application.cache import get_redis_instance
|
||||
from application.core.settings import settings
|
||||
from application.core.url_validation import SSRFError, validate_url
|
||||
from application.events.keys import stream_key
|
||||
from application.security.encryption import decrypt_credentials
|
||||
from docsgpt.agents.tools.base import Tool
|
||||
from docsgpt.api.user.tasks import mcp_oauth_task
|
||||
from docsgpt.cache import get_redis_instance
|
||||
from docsgpt.core.settings import settings
|
||||
from docsgpt.core.url_validation import SSRFError, validate_url
|
||||
from docsgpt.events.keys import stream_key
|
||||
from docsgpt.security.encryption import decrypt_credentials
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -856,10 +856,10 @@ class DBTokenStorage(TokenStorage):
|
||||
|
||||
def _fetch_session_data(self) -> dict:
|
||||
"""Read the JSONB ``session_data`` blob for this MCP server row."""
|
||||
from application.storage.db.repositories.connector_sessions import (
|
||||
from docsgpt.storage.db.repositories.connector_sessions import (
|
||||
ConnectorSessionsRepository,
|
||||
)
|
||||
from application.storage.db.session import db_readonly
|
||||
from docsgpt.storage.db.session import db_readonly
|
||||
|
||||
base_url = self.get_base_url(self.server_url)
|
||||
with db_readonly() as conn:
|
||||
@@ -893,10 +893,10 @@ class DBTokenStorage(TokenStorage):
|
||||
the scalar column — ``get_by_user_and_server_url`` needs that to
|
||||
resolve the row (``NULL = 'https://...'`` is UNKNOWN in SQL).
|
||||
"""
|
||||
from application.storage.db.repositories.connector_sessions import (
|
||||
from docsgpt.storage.db.repositories.connector_sessions import (
|
||||
ConnectorSessionsRepository,
|
||||
)
|
||||
from application.storage.db.session import db_session
|
||||
from docsgpt.storage.db.session import db_session
|
||||
|
||||
base_url = self.get_base_url(self.server_url)
|
||||
with db_session() as conn:
|
||||
@@ -905,10 +905,10 @@ class DBTokenStorage(TokenStorage):
|
||||
)
|
||||
|
||||
def _delete(self) -> None:
|
||||
from application.storage.db.repositories.connector_sessions import (
|
||||
from docsgpt.storage.db.repositories.connector_sessions import (
|
||||
ConnectorSessionsRepository,
|
||||
)
|
||||
from application.storage.db.session import db_session
|
||||
from docsgpt.storage.db.session import db_session
|
||||
|
||||
with db_session() as conn:
|
||||
ConnectorSessionsRepository(conn).delete(
|
||||
@@ -980,7 +980,7 @@ class DBTokenStorage(TokenStorage):
|
||||
"""
|
||||
from sqlalchemy import text
|
||||
|
||||
from application.storage.db.session import db_session
|
||||
from docsgpt.storage.db.session import db_session
|
||||
|
||||
def _delete_all() -> None:
|
||||
with db_session() as conn:
|
||||
@@ -4,8 +4,8 @@ import uuid
|
||||
|
||||
from .base import Tool
|
||||
from .path_utils import validate_tool_path
|
||||
from application.storage.db.repositories.memories import MemoriesRepository
|
||||
from application.storage.db.session import db_readonly, db_session
|
||||
from docsgpt.storage.db.repositories.memories import MemoriesRepository
|
||||
from docsgpt.storage.db.session import db_readonly, db_session
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -59,7 +59,7 @@ class MemoryTool(Tool):
|
||||
tool_id,
|
||||
)
|
||||
return False
|
||||
from application.storage.db.base_repository import looks_like_uuid
|
||||
from docsgpt.storage.db.base_repository import looks_like_uuid
|
||||
|
||||
if not looks_like_uuid(tool_id):
|
||||
logger.debug(
|
||||
@@ -2,8 +2,8 @@ from typing import Any, Dict, List, Optional
|
||||
import uuid
|
||||
|
||||
from .base import Tool
|
||||
from application.storage.db.repositories.notes import NotesRepository
|
||||
from application.storage.db.session import db_readonly, db_session
|
||||
from docsgpt.storage.db.repositories.notes import NotesRepository
|
||||
from docsgpt.storage.db.session import db_readonly, db_session
|
||||
|
||||
|
||||
# Stable synthetic title used in the Postgres ``notes.title`` column.
|
||||
@@ -55,7 +55,7 @@ class NotesTool(Tool):
|
||||
return False
|
||||
if tool_id.startswith("default_"):
|
||||
return False
|
||||
from application.storage.db.base_repository import looks_like_uuid
|
||||
from docsgpt.storage.db.base_repository import looks_like_uuid
|
||||
|
||||
return looks_like_uuid(tool_id)
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
from application.agents.tools.base import Tool
|
||||
from application.security.safe_url import UnsafeUserUrlError, pinned_request
|
||||
from docsgpt.agents.tools.base import Tool
|
||||
from docsgpt.security.safe_url import UnsafeUserUrlError, pinned_request
|
||||
|
||||
class NtfyTool(Tool):
|
||||
"""
|
||||
File renamed without changes.
@@ -2,7 +2,7 @@ import logging
|
||||
|
||||
import psycopg
|
||||
|
||||
from application.agents.tools.base import Tool
|
||||
from docsgpt.agents.tools.base import Tool
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -19,20 +19,20 @@ from typing import Any, Callable, Dict, List, Optional
|
||||
|
||||
from celery import current_task
|
||||
|
||||
from application.agents.tools.artifact_ref import resolve_artifact_id
|
||||
from application.agents.tools.attachment_bridge import (
|
||||
from docsgpt.agents.tools.artifact_ref import resolve_artifact_id
|
||||
from docsgpt.agents.tools.attachment_bridge import (
|
||||
AttachmentBridgeError,
|
||||
bridge_attachment,
|
||||
match_attachment,
|
||||
)
|
||||
from application.agents.tools.base import Tool
|
||||
from application.core.json_schema_utils import (
|
||||
from docsgpt.agents.tools.base import Tool
|
||||
from docsgpt.core.json_schema_utils import (
|
||||
JsonSchemaValidationError,
|
||||
normalize_json_schema_payload,
|
||||
)
|
||||
from application.core.settings import settings
|
||||
from application.storage.db.repositories.artifacts import ArtifactsRepository
|
||||
from application.storage.db.session import db_readonly
|
||||
from docsgpt.core.settings import settings
|
||||
from docsgpt.storage.db.repositories.artifacts import ArtifactsRepository
|
||||
from docsgpt.storage.db.session import db_readonly
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -223,7 +223,7 @@ class ReadDocumentTool(Tool):
|
||||
until the OUTER task's limit (webhook runs have none) kills the whole agent run.
|
||||
"""
|
||||
parent = self._parent()
|
||||
from application.api.user.tasks import parse_timeout_for_size
|
||||
from docsgpt.api.user.tasks import parse_timeout_for_size
|
||||
|
||||
# OCR cost scales with pages, so the parse window grows with the document's size
|
||||
# (floored at DOCUMENT_PARSE_TIMEOUT).
|
||||
@@ -232,7 +232,7 @@ class ReadDocumentTool(Tool):
|
||||
# ``current_task`` is a Celery proxy: truthy only while this runs inside a worker task,
|
||||
# falsy in the web process (the bare proxy is NOT identity-None, so test truthiness).
|
||||
if current_task:
|
||||
from application.worker import run_parse_document
|
||||
from docsgpt.worker import run_parse_document
|
||||
|
||||
try:
|
||||
result = self._run_inline_bounded(
|
||||
@@ -250,7 +250,7 @@ class ReadDocumentTool(Tool):
|
||||
|
||||
from celery.exceptions import TimeoutError as CeleryTimeoutError
|
||||
|
||||
from application.api.user.tasks import parse_document, parse_task_time_limits
|
||||
from docsgpt.api.user.tasks import parse_document, parse_task_time_limits
|
||||
|
||||
# The task's per-call time limits are raised to match the awaited window: bound to
|
||||
# the base timeout at import, the worker would otherwise self-terminate a large
|
||||
@@ -350,7 +350,7 @@ class ReadDocumentTool(Tool):
|
||||
# are non-daemon and registered with ``concurrent.futures``' atexit hook,
|
||||
# which joins them -- so an abandoned parse would hold up worker
|
||||
# shutdown for the rest of its (size-scaled) window. Same reasoning as
|
||||
# application/guardrails/engine.py. ``shutdown(cancel_futures=True)``
|
||||
# docsgpt/guardrails/engine.py. ``shutdown(cancel_futures=True)``
|
||||
# is not an alternative: it only drops queued work items, never the one
|
||||
# already running.
|
||||
slot: Dict[str, Any] = {}
|
||||
@@ -2,8 +2,8 @@ import codecs
|
||||
|
||||
from markdownify import markdownify
|
||||
|
||||
from application.agents.tools.base import Tool
|
||||
from application.security.safe_url import (
|
||||
from docsgpt.agents.tools.base import Tool
|
||||
from docsgpt.security.safe_url import (
|
||||
ResponseTooLargeError,
|
||||
UnsafeUserUrlError,
|
||||
pinned_fetch_bytes,
|
||||
@@ -11,18 +11,18 @@ import uuid
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any, Dict, Optional
|
||||
|
||||
from application.agents.tools.base import Tool
|
||||
from application.devices.broker import get_broker
|
||||
from application.devices.denylist import check_denylist
|
||||
from application.devices.normalizer import normalize_command
|
||||
from application.storage.db.repositories.device_audit_log import (
|
||||
from docsgpt.agents.tools.base import Tool
|
||||
from docsgpt.devices.broker import get_broker
|
||||
from docsgpt.devices.denylist import check_denylist
|
||||
from docsgpt.devices.normalizer import normalize_command
|
||||
from docsgpt.storage.db.repositories.device_audit_log import (
|
||||
DeviceAuditLogRepository,
|
||||
)
|
||||
from application.storage.db.repositories.device_auto_approve_patterns import (
|
||||
from docsgpt.storage.db.repositories.device_auto_approve_patterns import (
|
||||
DeviceAutoApprovePatternsRepository,
|
||||
)
|
||||
from application.storage.db.repositories.devices import DevicesRepository
|
||||
from application.storage.db.session import db_readonly, db_session
|
||||
from docsgpt.storage.db.repositories.devices import DevicesRepository
|
||||
from docsgpt.storage.db.session import db_readonly, db_session
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -7,16 +7,16 @@ import logging
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
from application.agents.scheduler_utils import (
|
||||
from docsgpt.agents.scheduler_utils import (
|
||||
ScheduleValidationError,
|
||||
clamp_once_horizon,
|
||||
parse_delay,
|
||||
parse_run_at,
|
||||
)
|
||||
from application.core.settings import settings
|
||||
from application.storage.db.base_repository import looks_like_uuid
|
||||
from application.storage.db.repositories.schedules import SchedulesRepository
|
||||
from application.storage.db.session import db_readonly, db_session
|
||||
from docsgpt.core.settings import settings
|
||||
from docsgpt.storage.db.base_repository import looks_like_uuid
|
||||
from docsgpt.storage.db.repositories.schedules import SchedulesRepository
|
||||
from docsgpt.storage.db.session import db_readonly, db_session
|
||||
|
||||
from .base import Tool
|
||||
|
||||
@@ -297,13 +297,13 @@ def _safe_default_allowlist(
|
||||
chat tools (resolved against ``settings.DEFAULT_CHAT_TOOLS`` and the
|
||||
user's ``tool_preferences.disabled_default_tools`` opt-outs).
|
||||
"""
|
||||
from application.agents.default_tools import (
|
||||
from docsgpt.agents.default_tools import (
|
||||
resolve_tool_by_id,
|
||||
synthesized_default_tools,
|
||||
)
|
||||
from application.storage.db.repositories.agents import AgentsRepository
|
||||
from application.storage.db.repositories.user_tools import UserToolsRepository
|
||||
from application.storage.db.repositories.users import UsersRepository
|
||||
from docsgpt.storage.db.repositories.agents import AgentsRepository
|
||||
from docsgpt.storage.db.repositories.user_tools import UserToolsRepository
|
||||
from docsgpt.storage.db.repositories.users import UsersRepository
|
||||
|
||||
def _is_safe(row: Dict[str, Any]) -> bool:
|
||||
actions = row.get("actions") or []
|
||||
File renamed without changes.
@@ -2,7 +2,7 @@ import logging
|
||||
|
||||
import requests
|
||||
|
||||
from application.agents.tools.base import Tool
|
||||
from docsgpt.agents.tools.base import Tool
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
from application.agents.tools.base import Tool
|
||||
from docsgpt.agents.tools.base import Tool
|
||||
|
||||
|
||||
THINK_TOOL_ID = "think"
|
||||
@@ -2,8 +2,8 @@ from typing import Any, Dict, List, Optional
|
||||
import uuid
|
||||
|
||||
from .base import Tool
|
||||
from application.storage.db.repositories.todos import TodosRepository
|
||||
from application.storage.db.session import db_readonly, db_session
|
||||
from docsgpt.storage.db.repositories.todos import TodosRepository
|
||||
from docsgpt.storage.db.session import db_readonly, db_session
|
||||
|
||||
|
||||
def _status_from_completed(completed: Any) -> str:
|
||||
@@ -60,7 +60,7 @@ class TodoListTool(Tool):
|
||||
return False
|
||||
if tool_id.startswith("default_"):
|
||||
return False
|
||||
from application.storage.db.base_repository import looks_like_uuid
|
||||
from docsgpt.storage.db.base_repository import looks_like_uuid
|
||||
|
||||
return looks_like_uuid(tool_id)
|
||||
|
||||
File renamed without changes.
@@ -3,7 +3,7 @@ import inspect
|
||||
import os
|
||||
import pkgutil
|
||||
|
||||
from application.agents.tools.base import Tool
|
||||
from docsgpt.agents.tools.base import Tool
|
||||
|
||||
|
||||
class ToolManager:
|
||||
@@ -17,7 +17,7 @@ class ToolManager:
|
||||
for finder, name, ispkg in pkgutil.iter_modules([tools_dir]):
|
||||
if name == "base" or name.startswith("__"):
|
||||
continue
|
||||
module = importlib.import_module(f"application.agents.tools.{name}")
|
||||
module = importlib.import_module(f"docsgpt.agents.tools.{name}")
|
||||
for member_name, obj in inspect.getmembers(module, inspect.isclass):
|
||||
if issubclass(obj, Tool) and obj is not Tool and not obj.internal:
|
||||
tool_config = self.config.get(name, {})
|
||||
@@ -25,7 +25,7 @@ class ToolManager:
|
||||
|
||||
def load_tool(self, tool_name, tool_config, user_id=None):
|
||||
self.config[tool_name] = tool_config
|
||||
module = importlib.import_module(f"application.agents.tools.{tool_name}")
|
||||
module = importlib.import_module(f"docsgpt.agents.tools.{tool_name}")
|
||||
for member_name, obj in inspect.getmembers(module, inspect.isclass):
|
||||
if issubclass(obj, Tool) and obj is not Tool:
|
||||
if (
|
||||
@@ -1,15 +1,15 @@
|
||||
import logging
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
from application.agents.tools.base import Tool
|
||||
from application.agents.tools.path_utils import validate_tool_path
|
||||
from application.storage.db.repositories.wiki_pages import (
|
||||
from docsgpt.agents.tools.base import Tool
|
||||
from docsgpt.agents.tools.path_utils import validate_tool_path
|
||||
from docsgpt.storage.db.repositories.wiki_pages import (
|
||||
WikiPageConflict,
|
||||
WikiPagesRepository,
|
||||
_content_hash,
|
||||
rebuild_wiki_directory_structure,
|
||||
)
|
||||
from application.storage.db.session import db_readonly, db_session
|
||||
from docsgpt.storage.db.session import db_readonly, db_session
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -207,7 +207,7 @@ class WikiTool(Tool):
|
||||
idempotency key guards each edit independently and dedups broker
|
||||
redeliveries without colliding across pages of the same source.
|
||||
"""
|
||||
from application.api.user.tasks import reembed_wiki_page
|
||||
from docsgpt.api.user.tasks import reembed_wiki_page
|
||||
|
||||
reembed_wiki_page.delay(
|
||||
self.source_id,
|
||||
@@ -2,11 +2,11 @@ import logging
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any, Dict, Generator, List, Optional, Tuple
|
||||
|
||||
from application.agents.base import BaseAgent
|
||||
from application.guardrails.config import (
|
||||
from docsgpt.agents.base import BaseAgent
|
||||
from docsgpt.guardrails.config import (
|
||||
DEFAULT_BLOCK_MESSAGE as GUARDRAIL_DEFAULT_MESSAGE,
|
||||
)
|
||||
from application.agents.workflows.schemas import (
|
||||
from docsgpt.agents.workflows.schemas import (
|
||||
ExecutionStatus,
|
||||
Workflow,
|
||||
WorkflowEdge,
|
||||
@@ -14,16 +14,16 @@ from application.agents.workflows.schemas import (
|
||||
WorkflowNode,
|
||||
WorkflowRun,
|
||||
)
|
||||
from application.agents.workflows.workflow_engine import WorkflowEngine
|
||||
from application.core.settings import settings
|
||||
from application.logging import LogContext, log_activity
|
||||
from application.sandbox.artifacts_capture import QuotaExceeded
|
||||
from application.storage.db.base_repository import looks_like_uuid
|
||||
from application.storage.db.repositories.workflow_edges import WorkflowEdgesRepository
|
||||
from application.storage.db.repositories.workflow_nodes import WorkflowNodesRepository
|
||||
from application.storage.db.repositories.workflow_runs import WorkflowRunsRepository
|
||||
from application.storage.db.repositories.workflows import WorkflowsRepository
|
||||
from application.storage.db.session import db_readonly, db_session
|
||||
from docsgpt.agents.workflows.workflow_engine import WorkflowEngine
|
||||
from docsgpt.core.settings import settings
|
||||
from docsgpt.logging import LogContext, log_activity
|
||||
from docsgpt.sandbox.artifacts_capture import QuotaExceeded
|
||||
from docsgpt.storage.db.base_repository import looks_like_uuid
|
||||
from docsgpt.storage.db.repositories.workflow_edges import WorkflowEdgesRepository
|
||||
from docsgpt.storage.db.repositories.workflow_nodes import WorkflowNodesRepository
|
||||
from docsgpt.storage.db.repositories.workflow_runs import WorkflowRunsRepository
|
||||
from docsgpt.storage.db.repositories.workflows import WorkflowsRepository
|
||||
from docsgpt.storage.db.session import db_readonly, db_session
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -327,8 +327,8 @@ class WorkflowAgent(BaseAgent):
|
||||
# parent), so skip the bridge for unowned/draft ids.
|
||||
if not persisted:
|
||||
return [], []
|
||||
from application.sandbox.artifacts_capture import persist_new_artifact
|
||||
from application.storage.storage_creator import StorageCreator
|
||||
from docsgpt.sandbox.artifacts_capture import persist_new_artifact
|
||||
from docsgpt.storage.storage_creator import StorageCreator
|
||||
|
||||
storage = StorageCreator.get_storage()
|
||||
max_bytes = int(getattr(settings, "ARTIFACT_MAX_BYTES", 0) or 0)
|
||||
File renamed without changes.
@@ -2,11 +2,11 @@
|
||||
|
||||
from typing import Dict, List, Optional, Type
|
||||
|
||||
from application.agents.agentic_agent import AgenticAgent
|
||||
from application.agents.base import BaseAgent
|
||||
from application.agents.classic_agent import ClassicAgent
|
||||
from application.agents.research_agent import ResearchAgent
|
||||
from application.agents.workflows.schemas import AgentType
|
||||
from docsgpt.agents.agentic_agent import AgenticAgent
|
||||
from docsgpt.agents.base import BaseAgent
|
||||
from docsgpt.agents.classic_agent import ClassicAgent
|
||||
from docsgpt.agents.research_agent import ResearchAgent
|
||||
from docsgpt.agents.workflows.schemas import AgentType
|
||||
|
||||
|
||||
class _WorkflowNodeMixin:
|
||||
File renamed without changes.
+36
-36
@@ -6,9 +6,9 @@ import uuid
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any, Dict, Generator, List, Optional, TYPE_CHECKING
|
||||
|
||||
from application.agents.workflows.cel_evaluator import CelEvaluationError, evaluate_cel
|
||||
from application.agents.workflows.node_agent import WorkflowNodeAgentFactory
|
||||
from application.agents.workflows.schemas import (
|
||||
from docsgpt.agents.workflows.cel_evaluator import CelEvaluationError, evaluate_cel
|
||||
from docsgpt.agents.workflows.node_agent import WorkflowNodeAgentFactory
|
||||
from docsgpt.agents.workflows.schemas import (
|
||||
AgentNodeConfig,
|
||||
AgentType,
|
||||
CodeNodeConfig,
|
||||
@@ -19,13 +19,13 @@ from application.agents.workflows.schemas import (
|
||||
WorkflowGraph,
|
||||
WorkflowNode,
|
||||
)
|
||||
from application.core.json_schema_utils import (
|
||||
from docsgpt.core.json_schema_utils import (
|
||||
JsonSchemaValidationError,
|
||||
normalize_json_schema_payload,
|
||||
)
|
||||
from application.error import sanitize_api_error
|
||||
from application.templates.namespaces import NamespaceManager
|
||||
from application.templates.template_engine import TemplateEngine, TemplateRenderError
|
||||
from docsgpt.error import sanitize_api_error
|
||||
from docsgpt.templates.namespaces import NamespaceManager
|
||||
from docsgpt.templates.template_engine import TemplateEngine, TemplateRenderError
|
||||
|
||||
try:
|
||||
import jsonschema
|
||||
@@ -33,7 +33,7 @@ except ImportError: # pragma: no cover - optional dependency in some deployment
|
||||
jsonschema = None
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from application.agents.base import BaseAgent
|
||||
from docsgpt.agents.base import BaseAgent
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
StateValue = Any
|
||||
@@ -101,7 +101,7 @@ class WorkflowEngine:
|
||||
# node and agent-node tool in this run, so it is torn down exactly once
|
||||
# here rather than per node. peek_manager() never builds the manager, so
|
||||
# a run that never opened a session closes nothing.
|
||||
from application.sandbox.sandbox_creator import SandboxCreator
|
||||
from docsgpt.sandbox.sandbox_creator import SandboxCreator
|
||||
|
||||
mgr = SandboxCreator.peek_manager()
|
||||
if mgr is not None:
|
||||
@@ -312,13 +312,13 @@ class WorkflowEngine:
|
||||
def _execute_agent_node(
|
||||
self, node: WorkflowNode
|
||||
) -> Generator[Dict[str, str], None, None]:
|
||||
from application.core.model_utils import (
|
||||
from docsgpt.core.model_utils import (
|
||||
get_api_key_for_provider,
|
||||
get_model_capabilities,
|
||||
resolve_dispatch_provider,
|
||||
)
|
||||
|
||||
from application.api.answer.services.prompt_renderer import (
|
||||
from docsgpt.api.answer.services.prompt_renderer import (
|
||||
prompt_embeds_documents as _prompt_embeds_documents,
|
||||
)
|
||||
|
||||
@@ -512,8 +512,8 @@ class WorkflowEngine:
|
||||
self, node: WorkflowNode
|
||||
) -> Generator[Dict[str, str], None, None]:
|
||||
"""Run code in the run-scoped sandbox, persist produced files, and write an artifact reference."""
|
||||
from application.sandbox.artifacts_capture import capture_artifacts, snapshot_signatures
|
||||
from application.sandbox.sandbox_creator import SandboxCreator
|
||||
from docsgpt.sandbox.artifacts_capture import capture_artifacts, snapshot_signatures
|
||||
from docsgpt.sandbox.sandbox_creator import SandboxCreator
|
||||
|
||||
config = CodeNodeConfig(**node.config.get("config", node.config))
|
||||
code = config.code or ""
|
||||
@@ -627,13 +627,13 @@ class WorkflowEngine:
|
||||
self, manager: Any, session_id: str, inputs: List[str], user_id: str
|
||||
) -> List[str]:
|
||||
"""Stage referenced input artifacts (run-scoped, never cross-tenant) into the workspace."""
|
||||
from application.agents.tools.artifact_ref import resolve_artifact_id
|
||||
from application.core.settings import settings
|
||||
from application.sandbox.artifacts_capture import unique_input_path
|
||||
from application.storage.db.repositories.artifacts import ArtifactsRepository
|
||||
from application.storage.db.session import db_readonly
|
||||
from application.storage.storage_creator import StorageCreator
|
||||
from application.utils import safe_filename
|
||||
from docsgpt.agents.tools.artifact_ref import resolve_artifact_id
|
||||
from docsgpt.core.settings import settings
|
||||
from docsgpt.sandbox.artifacts_capture import unique_input_path
|
||||
from docsgpt.storage.db.repositories.artifacts import ArtifactsRepository
|
||||
from docsgpt.storage.db.session import db_readonly
|
||||
from docsgpt.storage.storage_creator import StorageCreator
|
||||
from docsgpt.utils import safe_filename
|
||||
|
||||
loaded: List[str] = []
|
||||
raw_ids = self._resolve_input_artifact_ids(inputs)
|
||||
@@ -693,9 +693,9 @@ class WorkflowEngine:
|
||||
selection, so it never widens per-node document access. Best-effort:
|
||||
a resolution failure drops the manifest, never the node.
|
||||
"""
|
||||
from application.agents.tools.artifact_ref import make_ref, resolve_artifact_id
|
||||
from application.storage.db.repositories.artifacts import ArtifactsRepository
|
||||
from application.storage.db.session import db_readonly
|
||||
from docsgpt.agents.tools.artifact_ref import make_ref, resolve_artifact_id
|
||||
from docsgpt.storage.db.repositories.artifacts import ArtifactsRepository
|
||||
from docsgpt.storage.db.session import db_readonly
|
||||
|
||||
try:
|
||||
raw_ids = self._resolve_input_artifact_ids(node_config.input_documents)
|
||||
@@ -738,10 +738,10 @@ class WorkflowEngine:
|
||||
supported_types: List[str],
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""Resolve a node's selected documents to native/extracted attachment dicts for its LLM."""
|
||||
from application.agents.tools.artifact_ref import resolve_artifact_id
|
||||
from application.core.settings import settings
|
||||
from application.storage.db.repositories.artifacts import ArtifactsRepository
|
||||
from application.storage.db.session import db_readonly
|
||||
from docsgpt.agents.tools.artifact_ref import resolve_artifact_id
|
||||
from docsgpt.core.settings import settings
|
||||
from docsgpt.storage.db.repositories.artifacts import ArtifactsRepository
|
||||
from docsgpt.storage.db.session import db_readonly
|
||||
|
||||
raw_ids = self._resolve_input_artifact_ids(node_config.input_documents)
|
||||
if not raw_ids:
|
||||
@@ -909,8 +909,8 @@ class WorkflowEngine:
|
||||
deadline: ``time.monotonic()`` value past which no further blocking
|
||||
parse may run, shared across the node's documents.
|
||||
"""
|
||||
from application.parser.document_reader import truncate_text_head_tail
|
||||
from application.storage.storage_creator import StorageCreator
|
||||
from docsgpt.parser.document_reader import truncate_text_head_tail
|
||||
from docsgpt.storage.storage_creator import StorageCreator
|
||||
|
||||
# An uploaded chat attachment was already parsed (and possibly OCR'd) when it
|
||||
# was stored, so re-parsing it here would repeat the dominant cost of the run
|
||||
@@ -965,12 +965,12 @@ class WorkflowEngine:
|
||||
"""
|
||||
from celery.exceptions import TimeoutError as CeleryTimeoutError
|
||||
|
||||
from application.api.user.tasks import (
|
||||
from docsgpt.api.user.tasks import (
|
||||
parse_document,
|
||||
parse_task_time_limits,
|
||||
parse_timeout_for_size,
|
||||
)
|
||||
from application.core.settings import settings
|
||||
from docsgpt.core.settings import settings
|
||||
|
||||
user_id = self._resolve_user_id()
|
||||
if not user_id:
|
||||
@@ -999,7 +999,7 @@ class WorkflowEngine:
|
||||
# default ``disable_sync_subtasks=True`` makes ``get()`` raise RuntimeError
|
||||
# ("Never call result.get() within a task!"). The dedicated parsing queue +
|
||||
# separate workers already avoid the real self-deadlock, so opt out
|
||||
# explicitly (mirrors application/agents/tools/read_document.py).
|
||||
# explicitly (mirrors docsgpt/agents/tools/read_document.py).
|
||||
result = async_result.get(timeout=timeout, disable_sync_subtasks=False)
|
||||
except (CeleryTimeoutError, TimeoutError):
|
||||
logger.warning("Workflow node: document parse timed out for %s", artifact_id)
|
||||
@@ -1082,7 +1082,7 @@ class WorkflowEngine:
|
||||
|
||||
def _resolve_code_timeout(self, requested: Optional[int]) -> float:
|
||||
"""Return the stricter of the node's requested timeout and the sandbox cap."""
|
||||
from application.core.settings import settings
|
||||
from docsgpt.core.settings import settings
|
||||
|
||||
cap = float(getattr(settings, "SANDBOX_EXEC_TIMEOUT", 60))
|
||||
if requested is None:
|
||||
@@ -1331,8 +1331,8 @@ class WorkflowEngine:
|
||||
logger.warning("Workflow node sources dropped: no owner to authorize.")
|
||||
return []
|
||||
|
||||
from application.api.user.team_sharing import can_access
|
||||
from application.storage.db.session import db_readonly
|
||||
from docsgpt.api.user.team_sharing import can_access
|
||||
from docsgpt.storage.db.session import db_readonly
|
||||
|
||||
allowed = []
|
||||
try:
|
||||
@@ -1359,7 +1359,7 @@ class WorkflowEngine:
|
||||
Returns:
|
||||
list: Retrieved documents, empty when there was nothing to fetch.
|
||||
"""
|
||||
from application.retriever.retriever_creator import RetrieverCreator
|
||||
from docsgpt.retriever.retriever_creator import RetrieverCreator
|
||||
|
||||
query = self.state.get("query", "")
|
||||
if not query:
|
||||
@@ -1,11 +1,11 @@
|
||||
# Alembic configuration for the DocsGPT user-data Postgres database.
|
||||
#
|
||||
# The SQLAlchemy URL is deliberately NOT set here — env.py reads it from
|
||||
# ``application.core.settings.settings.POSTGRES_URI`` so the same config
|
||||
# ``docsgpt.core.settings.settings.POSTGRES_URI`` so the same config
|
||||
# source serves the running app and migrations. To run from the project
|
||||
# root::
|
||||
#
|
||||
# alembic -c application/alembic.ini upgrade head
|
||||
# alembic -c docsgpt/alembic.ini upgrade head
|
||||
|
||||
[alembic]
|
||||
script_location = %(here)s/alembic
|
||||
@@ -1,6 +1,6 @@
|
||||
"""Alembic environment for the DocsGPT user-data Postgres database.
|
||||
|
||||
The URL is pulled from ``application.core.settings`` rather than
|
||||
The URL is pulled from ``docsgpt.core.settings`` rather than
|
||||
``alembic.ini`` so that a single ``POSTGRES_URI`` env var drives both the
|
||||
running app and ``alembic`` CLI invocations.
|
||||
"""
|
||||
@@ -10,7 +10,7 @@ from logging.config import fileConfig
|
||||
from pathlib import Path
|
||||
|
||||
# Make the project root importable regardless of cwd. env.py lives at
|
||||
# <repo>/application/alembic/env.py, so parents[2] is the repo root.
|
||||
# <repo>/docsgpt/alembic/env.py, so parents[2] is the repo root.
|
||||
_PROJECT_ROOT = Path(__file__).resolve().parents[2]
|
||||
if str(_PROJECT_ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(_PROJECT_ROOT))
|
||||
@@ -18,8 +18,8 @@ if str(_PROJECT_ROOT) not in sys.path:
|
||||
from alembic import context # noqa: E402
|
||||
from sqlalchemy import engine_from_config, pool # noqa: E402
|
||||
|
||||
from application.core.settings import settings # noqa: E402
|
||||
from application.storage.db.models import metadata as target_metadata # noqa: E402
|
||||
from docsgpt.core.settings import settings # noqa: E402
|
||||
from docsgpt.storage.db.models import metadata as target_metadata # noqa: E402
|
||||
|
||||
config = context.config
|
||||
|
||||
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
Loaded 100 of 1006 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user