Merge pull request #2743 from arc53/feat/pypi-package

feat: publish the backend to PyPI as docsgpt

.github/workflows/package-build.yml

@@ -0,0 +1,88 @@
+name: Build the Python package
+
+# Builds the sdist and wheel, checks the wheel carries the data the app reads
+# at runtime and nothing from the checkout, then installs it into a clean
+# environment and runs the docsgpt command.
+
+on:
+  pull_request:
+    paths:
+      - 'pyproject.toml'
+      - 'uv.lock'
+      - 'docsgpt/**'
+      - 'README.md'
+      - 'LICENSE'
+      - '.github/workflows/package-build.yml'
+  push:
+    branches: [main]
+    paths:
+      - 'pyproject.toml'
+      - 'uv.lock'
+      - 'docsgpt/**'
+      - 'README.md'
+      - 'LICENSE'
+      - '.github/workflows/package-build.yml'
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
+        with:
+          persist-credentials: false
+
+      - uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6.8.0
+
+      - name: Build the sdist and wheel
+        run: uv build
+
+      - name: Check the archives
+        run: |
+          uvx twine check --strict dist/*
+          python3 - <<'PY'
+          import glob
+          import zipfile
+
+          names = set(zipfile.ZipFile(glob.glob("dist/*.whl")[0]).namelist())
+          for required in (
+              "docsgpt/cli.py",
+              "docsgpt/alembic.ini",
+              "docsgpt/alembic/env.py",
+              "docsgpt/alembic/script.py.mako",
+              "docsgpt/prompts/chat_reduce_prompt.txt",
+              "docsgpt/prompts/fragments/identity.txt",
+              "docsgpt/core/models/_defaults.yaml",
+              "docsgpt/seed/config/premade_agents.yaml",
+          ):
+              assert required in names, f"missing from the wheel: {required}"
+          for banned in (
+              "docsgpt/Dockerfile",
+              "docsgpt/requirements.txt",
+              "docsgpt/requirements-docling.txt",
+              "docsgpt/index.faiss",
+              "docsgpt/index.pkl",
+              "application/__init__.py",
+          ):
+              assert banned not in names, f"must not ship: {banned}"
+          stray = [n for n in names if n.startswith(("docsgpt/indexes/", "docsgpt/inputs/", "docsgpt/vectors/"))]
+          assert not stray, stray
+          print(f"wheel ok: {len(names)} files")
+          PY
+
+      - name: Install the wheel into a clean environment and run the command
+        run: |
+          uv venv "$RUNNER_TEMP/docsgpt-venv"
+          uv pip install --python "$RUNNER_TEMP/docsgpt-venv/bin/python" dist/*.whl
+          "$RUNNER_TEMP/docsgpt-venv/bin/docsgpt" --version
+          "$RUNNER_TEMP/docsgpt-venv/bin/docsgpt" --help
+          "$RUNNER_TEMP/docsgpt-venv/bin/docsgpt" api --help
+          "$RUNNER_TEMP/docsgpt-venv/bin/python" -c "from docsgpt.core import paths; print('home:', paths.home_dir())"
+
+      - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: dist
+          path: dist/

.github/workflows/pypi-publish.yml

@@ -0,0 +1,86 @@
+name: Publish to PyPI
+
+# Trusted publishing: PyPI trusts this workflow file in arc53/DocsGPT running
+# in the `pypi` environment, so no API token is stored. A published release
+# (which the backend-release workflow creates when docsgpt/version.py changes
+# on main) publishes to PyPI; a manual run publishes to TestPyPI by default.
+
+on:
+  release:
+    types: [published]
+  workflow_dispatch:
+    inputs:
+      target:
+        description: Package index to publish to
+        type: choice
+        options: [testpypi, pypi]
+        default: testpypi
+
+permissions:
+  contents: read
+
+jobs:
+  build:
+    if: github.repository == 'arc53/DocsGPT'
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
+        with:
+          persist-credentials: false
+
+      - uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6.8.0
+        with:
+          # A publishing workflow must not restore a cache another job could have poisoned.
+          enable-cache: false
+
+      - name: Build the sdist and wheel
+        run: uv build
+
+      - name: The archives carry the released version
+        if: github.event_name == 'release'
+        env:
+          TAG: ${{ github.event.release.tag_name }}
+        run: |
+          ls "dist/docsgpt-${TAG}.tar.gz" "dist/docsgpt-${TAG}-py3-none-any.whl"
+
+      - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: dist
+          path: dist/
+          if-no-files-found: error
+
+  publish-pypi:
+    if: github.event_name == 'release' || inputs.target == 'pypi'
+    needs: build
+    runs-on: ubuntu-latest
+    environment:
+      name: pypi
+      url: https://pypi.org/project/docsgpt/
+    permissions:
+      id-token: write
+    steps:
+      - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+        with:
+          name: dist
+          path: dist/
+
+      - uses: pypa/gh-action-pypi-publish@dc37677b2e1c63e2034f94d8a5b11f265b73ba33 # v1.14.2
+
+  publish-testpypi:
+    if: github.event_name == 'workflow_dispatch' && inputs.target == 'testpypi'
+    needs: build
+    runs-on: ubuntu-latest
+    environment:
+      name: testpypi
+      url: https://test.pypi.org/project/docsgpt/
+    permissions:
+      id-token: write
+    steps:
+      - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+        with:
+          name: dist
+          path: dist/
+
+      - uses: pypa/gh-action-pypi-publish@dc37677b2e1c63e2034f94d8a5b11f265b73ba33 # v1.14.2
+        with:
+          repository-url: https://test.pypi.org/legacy/

.gitignore

@@ -206,3 +206,6 @@ tests/e2e/.e2e-last-run.json
 docsgpt/core/models/foundry_test_internal.yaml
 bench/
 /benchmark/
+
+# Python package build output (uv build)
+dist/

AGENTS.md

@@ -37,6 +37,14 @@ uv pip install -r docsgpt/requirements.txt  # or: pip install -r docsgpt/require
 # (the file adds the PyTorch CPU index; prefer `uv sync --extra docling`).
 ```
 
+The backend is also an installable package (`pyproject.toml`, hatchling).
+`uv sync` installs it editable and puts a `docsgpt` command on PATH:
+`docsgpt api --reload`, `docsgpt worker`, `docsgpt migrate`,
+`docsgpt prefetch-models`, `docsgpt verify-offline`. Runtime data (`.env`,
+`inputs/`, `indexes/`) lives in the checkout by default; `DOCSGPT_HOME` moves
+that data home, and `DOCSGPT_ENV_FILE` selects only the `.env` file (see
+`docsgpt/core/paths.py`).
+
 Dependencies are declared in `pyproject.toml` and locked in `uv.lock`; the
 `docsgpt/requirements*.txt` files are exported from the lock. To add or
 bump a package: edit `pyproject.toml`, run `uv lock`, then

docs/content/Deploying/Pip-Install.mdx

@@ -0,0 +1,101 @@
+---
+title: Install with pip
+description: Run the DocsGPT backend from the PyPI package, inside your own Python environment.
+---
+
+import { Callout } from 'nextra/components'
+
+# Install with pip
+
+The DocsGPT backend is on PyPI as [`docsgpt`](https://pypi.org/project/docsgpt/): the API server, the Celery worker and the maintenance scripts in one package, behind a single `docsgpt` command. Use it when you want DocsGPT inside your own Python environment or process manager rather than the [Docker images](/Deploying/Docker-Deploying).
+
+<Callout type="info">
+  The package is the backend only. The web UI ships in the Docker images and in the repository under `frontend/`; serving it from the package is planned.
+</Callout>
+
+## Requirements
+
+- Python 3.12 or newer
+- [PostgreSQL](/Deploying/Postgres-Migration) for user data, with the `vector` extension if you set `VECTOR_STORE=pgvector`
+- Redis for the task queue and the cache
+- An LLM: an API key for a hosted provider, or a local model server
+
+## Install
+
+```bash
+python -m venv .venv && source .venv/bin/activate
+pip install docsgpt
+```
+
+Extras add the optional engines:
+
+```bash
+pip install "docsgpt[docling]"   # DOC_PARSER_ENGINE=docling: OCR backend, structured output
+pip install "docsgpt[milvus]"    # VECTOR_STORE=milvus
+```
+
+The `docling` extra pulls in PyTorch, and on Linux the PyPI torch wheels bring the CUDA stack with them. On a CPU-only machine install the CPU build first, then the extra; pip keeps the torch it already has:
+
+```bash
+pip install --index-url https://download.pytorch.org/whl/cpu torch torchvision
+pip install "docsgpt[docling]"
+```
+
+`uv pip install` accepts the same two commands. With `pipx`, install `docsgpt[docling]` first, then replace the CUDA build inside its environment (`--no-deps` keeps pip from touching torch's dependencies, which the PyTorch index carries in older copies):
+
+```bash
+pipx runpip docsgpt install --force-reinstall --no-deps --index-url https://download.pytorch.org/whl/cpu torch torchvision
+```
+
+## Configure
+
+DocsGPT keeps its runtime files in a **data home**: the `.env` file it reads settings from, uploaded files under `inputs/` and vector indexes under `indexes/`. The data home is the directory you run the commands from, or the directory `DOCSGPT_HOME` points to. `DOCSGPT_ENV_FILE` points at a `.env` kept somewhere else.
+
+Create a `.env` in the data home. The minimum for a hosted LLM:
+
+```ini
+# LLM provider and model: see App Configuration for the options
+LLM_PROVIDER=openai
+LLM_NAME=<model name>
+API_KEY=<provider API key>
+
+# User data
+POSTGRES_URI=postgresql://docsgpt:<password>@localhost:5432/docsgpt
+
+# Worker-to-API authentication (required for uploads)
+INTERNAL_KEY=<a long random string>
+```
+
+Redis defaults to `localhost:6379` (`CELERY_BROKER_URL`, `CELERY_RESULT_BACKEND`, `CACHE_REDIS_URL`). Every setting is listed in [App Configuration](/Deploying/DocsGPT-Settings).
+
+## Run
+
+```bash
+docsgpt migrate    # create the database if it is missing and apply the migrations
+docsgpt api        # the API on http://127.0.0.1:7091
+docsgpt worker     # in a second terminal: the Celery worker, with the scheduler
+```
+
+`docsgpt api` listens on localhost only. Pass `--host 0.0.0.0` to accept connections from other machines or containers, and put a reverse proxy with TLS in front of it for anything public.
+
+The API applies pending migrations when it starts (`AUTO_MIGRATE`), so `docsgpt migrate` is the explicit step for deployments that want the schema in place before the first request or that run the API with a restricted database role.
+
+Both commands print the data home they resolved on start-up. Run them from the same directory, or set `DOCSGPT_HOME` for both, so the worker finds the files the API stores and the API finds the indexes the worker builds.
+
+The worker is not optional: query embedding runs on it, so search fails without one. `docsgpt worker --help` lists the queue, concurrency and pool options; `--no-beat` starts a worker without the scheduler when another worker already runs it. On Windows the scheduler cannot be embedded, so run `docsgpt beat` in a third terminal.
+
+Other commands:
+
+- `docsgpt api --reload`: a development server with auto-reload.
+- `docsgpt prefetch-models`: download the embedding, tokenizer and parser models ahead of time, for machines that go offline.
+- `docsgpt verify-offline`: check that a prepared install starts with networking off.
+- `docsgpt reembed`: re-embed every index after changing `EMBEDDINGS_NAME` (see [Upgrading](/upgrading)).
+
+## Upgrade
+
+```bash
+pip install -U docsgpt
+docsgpt migrate
+```
+
+Read the [upgrade notes](/upgrading) first when moving between minor versions.

docs/content/Deploying/_meta.js

@@ -15,6 +15,10 @@ export default {
     "title": "🛳️ Docker Setup",
     "href": "/Deploying/Docker-Deploying"
   },
+  "Pip-Install": {
+    "title": "🐍 Install with pip",
+    "href": "/Deploying/Pip-Install"
+  },
   "Development-Environment": {
     "title": "🛠️Development Environment",
     "href": "/Deploying/Development-Environment"

docs/content/upgrading.mdx

@@ -111,6 +111,16 @@ alias, so nothing breaks on upgrade, but update these before the alias goes:
   than it would have (a monthly sync by up to a month). Nothing to do.
 - Data directories do not move: the compose files keep your indexes, inputs
   and vectors under `application/` in the checkout, where they already are.
+- The backend is also a package now (`pip install docsgpt`, see
+  [Install with pip](/Deploying/Pip-Install)). Runtime data lives in a data
+  home: `DOCSGPT_HOME`, else the checkout, else the directory the process
+  starts in. One consequence for a source checkout: the embedded Milvus
+  (`MILVUS_URI`) and LanceDB (`LANCEDB_PATH`) default paths now resolve under
+  the checkout instead of the start directory. If you use either store at its
+  default path and start DocsGPT from another directory, the old data is at
+  `<start directory>/milvus_local.db` or `<start directory>/data/lancedb`;
+  point the setting at it, or move it into the checkout. Faiss indexes and
+  uploads were already stored under the checkout and are unaffected.
 
 ## Check your version
 

docsgpt/Dockerfile

@@ -150,7 +150,10 @@ ENV FLASK_APP=app.py
 # Thread caps. onnxruntime (FastEmbed) ignores OMP_NUM_THREADS and sizes its
 # pool to the host's core count, which a CPU-limited container still reports;
 # EMBEDDINGS_THREADS pins it the way OMP_NUM_THREADS pinned torch before.
-ENV MALLOC_ARENA_MAX=2 \
+# The image ships no checkout, so pin the data home (inputs, indexes, .env)
+# instead of depending on the working directory.
+ENV DOCSGPT_HOME=/app \
+    MALLOC_ARENA_MAX=2 \
     OMP_NUM_THREADS=4 \
     MKL_NUM_THREADS=4 \
     OPENBLAS_NUM_THREADS=4 \

docsgpt/api/internal/routes.py

@@ -5,6 +5,7 @@ from flask import Blueprint, request, send_from_directory, jsonify
 from werkzeug.utils import secure_filename
 import logging
 
+from docsgpt.core.paths import home_dir
 from docsgpt.core.settings import settings
 from docsgpt.storage.db.base_repository import looks_like_uuid
 from docsgpt.storage.db.repositories.sources import SourcesRepository
@@ -14,9 +15,7 @@ from docsgpt.storage.storage_creator import StorageCreator
 
 logger = logging.getLogger(__name__)
 
-current_dir = os.path.dirname(
-    os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
-)
+current_dir = str(home_dir())
 
 
 internal = Blueprint("internal", __name__)

docsgpt/api/user/base.py

@@ -32,10 +32,6 @@ from docsgpt.vectorstore.vector_creator import VectorCreator
 storage = StorageCreator.get_storage()
 
 
-current_dir = os.path.dirname(
-    os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
-)
-
 def generate_minute_range(start_date, end_date):
     """Generate a dictionary with minute-level time ranges."""
     return {

docsgpt/app.py

@@ -11,6 +11,7 @@ from werkzeug.exceptions import RequestEntityTooLarge
 from docsgpt.auth import handle_auth
 
 from docsgpt.core import log_context
+from docsgpt.core.paths import env_file
 from docsgpt.core.logging_config import setup_logging
 
 setup_logging()
@@ -53,7 +54,7 @@ if platform.system() == "Windows":
     import pathlib
 
     pathlib.PosixPath = pathlib.WindowsPath
-dotenv.load_dotenv()
+dotenv.load_dotenv(env_file())
 
 # Self-bootstrap the user-data Postgres DB. Runs before any blueprint or
 # repository touches the engine, so the first request can't race the

docsgpt/cli.py

@@ -0,0 +1,180 @@
+"""The ``docsgpt`` command: run the API, the worker and the maintenance scripts.
+
+Every subcommand imports what it needs when it runs, so ``docsgpt --help``
+stays instant and does not touch the database.
+"""
+
+from __future__ import annotations
+
+import argparse
+import logging
+import os
+import sys
+from typing import Optional, Sequence
+
+from docsgpt.version import __version__
+
+DEFAULT_HOST = "127.0.0.1"
+DEFAULT_PORT = 7091
+
+
+def _announce_home() -> None:
+    """Say where runtime data and the env file come from; the API and the worker must agree."""
+    from docsgpt.core.paths import env_file, home_dir
+
+    print(f"docsgpt: data home {home_dir()} (env file {env_file()})", file=sys.stderr)
+
+
+def _api(args: argparse.Namespace) -> int:
+    """Serve the ASGI app: gunicorn with the bounded-drain uvicorn worker, or uvicorn when reloading."""
+    _announce_home()
+    if args.reload or sys.platform == "win32":
+        import uvicorn
+
+        uvicorn.run("docsgpt.asgi:asgi_app", host=args.host, port=args.port, reload=args.reload)
+        return 0
+
+    from gunicorn.app.wsgiapp import run as gunicorn_run
+
+    argv = [
+        "gunicorn",
+        "-w", str(args.workers),
+        "-k", "docsgpt.gunicorn_worker.BoundedDrainUvicornWorker",
+        "--bind", f"{args.host}:{args.port}",
+        "--timeout", "180",
+        "--graceful-timeout", "120",
+        "--keep-alive", "5",
+        "--max-requests", "5000",
+        "--max-requests-jitter", "500",
+        "--config", "python:docsgpt.gunicorn_conf",
+    ]
+    if os.path.isdir("/dev/shm"):
+        argv += ["--worker-tmp-dir", "/dev/shm"]
+    argv.append("docsgpt.asgi:asgi_app")
+    sys.argv = argv
+    gunicorn_run()
+    return 0
+
+
+def _celery(argv: list[str]) -> int:
+    """Run a celery subcommand on the app and return its exit code (usage errors print usage)."""
+    import click
+
+    from docsgpt.app import celery
+
+    try:
+        code = celery.start(argv)
+    except click.ClickException as exc:
+        exc.show()
+        return exc.exit_code
+    return int(code or 0)
+
+
+def _worker(args: argparse.Namespace) -> int:
+    """Run the Celery worker, with the beat scheduler embedded unless ``--no-beat`` (or on Windows)."""
+    _announce_home()
+    windows = sys.platform == "win32"
+    argv = ["worker", "-l", args.loglevel]
+    if args.queues:
+        argv += ["-Q", args.queues]
+    if args.concurrency:
+        argv += ["--concurrency", str(args.concurrency)]
+    if args.beat and windows:
+        print("docsgpt: the embedded scheduler is not available on Windows; run `docsgpt beat` separately.", file=sys.stderr)
+    elif args.beat:
+        argv.append("-B")
+    pool = args.pool or ("solo" if sys.platform in ("darwin", "win32") else None)
+    if pool:
+        argv += ["--pool", pool]
+    return _celery(argv)
+
+
+def _beat(args: argparse.Namespace) -> int:
+    """Run the beat scheduler on its own (Windows, or a worker started with ``--no-beat``)."""
+    _announce_home()
+    return _celery(["beat", "-l", args.loglevel])
+
+
+def _migrate(args: argparse.Namespace) -> int:
+    """Create the Postgres database if asked and migrate it to head."""
+    from docsgpt.core.settings import settings
+    from docsgpt.storage.db.bootstrap import ensure_database_ready
+
+    _announce_home()
+    logging.basicConfig(level=logging.INFO, format="%(levelname)s %(name)s: %(message)s")
+    if not settings.POSTGRES_URI:
+        print("POSTGRES_URI is not set; nothing to migrate.", file=sys.stderr)
+        return 2
+    ensure_database_ready(
+        settings.POSTGRES_URI,
+        create_db=args.create_db,
+        migrate=True,
+        logger=logging.getLogger("docsgpt.migrate"),
+    )
+    return 0
+
+
+# Maintenance scripts keep their own argument parsers; the command hands
+# everything after the script name to them untouched (argparse would try to
+# interpret the options itself).
+SCRIPTS = {
+    "prefetch-models": ("prefetch_models", "download the embedding, tokenizer and parser models"),
+    "verify-offline": ("verify_offline", "check that the install runs with networking off"),
+    "reembed": ("reembed", "re-embed every index with the configured embedding model"),
+}
+
+
+def _run_script(module: str, argv: list[str]) -> int:
+    import importlib
+
+    return int(importlib.import_module(f"docsgpt.scripts.{module}").main(argv) or 0)
+
+
+def build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(prog="docsgpt", description="DocsGPT: private AI for agents, assistants and search.")
+    parser.add_argument("--version", action="version", version=f"docsgpt {__version__}")
+    commands = parser.add_subparsers(dest="command", metavar="<command>")
+
+    api = commands.add_parser("api", help="serve the HTTP API")
+    api.add_argument("--host", default=DEFAULT_HOST, help="interface to listen on (default: localhost; 0.0.0.0 for all)")
+    api.add_argument("--port", type=int, default=DEFAULT_PORT)
+    api.add_argument("--workers", type=int, default=1, help="gunicorn worker processes (default: 1)")
+    api.add_argument("--reload", action="store_true", help="development mode: uvicorn with auto-reload")
+    api.set_defaults(func=_api)
+
+    worker = commands.add_parser("worker", help="run the Celery worker (and the scheduler)")
+    worker.add_argument("-Q", "--queues", help="queues to consume (default: every configured queue)")
+    worker.add_argument("--concurrency", type=int, help="worker processes (default: one per CPU)")
+    worker.add_argument("--pool", help="celery pool (default: prefork; solo on macOS and Windows)")
+    worker.add_argument("-l", "--loglevel", default="INFO")
+    worker.add_argument("--no-beat", dest="beat", action="store_false", help="do not embed the beat scheduler")
+    worker.set_defaults(func=_worker)
+
+    beat = commands.add_parser("beat", help="run the beat scheduler on its own")
+    beat.add_argument("-l", "--loglevel", default="INFO")
+    beat.set_defaults(func=_beat)
+
+    migrate = commands.add_parser("migrate", help="create the database if needed and run the migrations")
+    migrate.add_argument("--no-create", dest="create_db", action="store_false", help="fail instead of creating a missing database")
+    migrate.set_defaults(func=_migrate)
+
+    for name, (module, help_text) in SCRIPTS.items():
+        commands.add_parser(name, help=f"{help_text} (docsgpt.scripts.{module})", add_help=False)
+
+    return parser
+
+
+def main(argv: Optional[Sequence[str]] = None) -> int:
+    argv = list(sys.argv[1:] if argv is None else argv)
+    if argv and argv[0] in SCRIPTS:
+        return _run_script(SCRIPTS[argv[0]][0], argv[1:])
+    parser = build_parser()
+    args = parser.parse_args(argv)
+    if not args.command:
+        parser.print_help()
+        return 2
+    return args.func(args)
+
+
+if __name__ == "__main__":
+    sys.exit(main())

docsgpt/core/paths.py

@@ -0,0 +1,50 @@
+"""Where DocsGPT keeps its files.
+
+Three locations matter. The package directory holds code and the data that
+ships with it (prompts, model catalogs, migrations). A checkout, when the
+package is imported from one, is the directory holding ``pyproject.toml``.
+The data home is where runtime data lives: ``.env``, ``inputs``, ``indexes``.
+
+The home is ``DOCSGPT_HOME`` when set, else the checkout, else the current
+directory. That keeps a source checkout and the Docker image (which runs from
+``/app`` with the package beside it) behaving as before, and gives a
+``pip install docsgpt`` user a home that is not ``site-packages``.
+"""
+
+from __future__ import annotations
+
+import os
+from pathlib import Path
+
+HOME_ENV = "DOCSGPT_HOME"
+ENV_FILE_ENV = "DOCSGPT_ENV_FILE"
+
+
+def package_dir() -> Path:
+    """The installed ``docsgpt`` package directory."""
+    return Path(__file__).resolve().parent.parent
+
+
+def checkout_root() -> Path | None:
+    """The source checkout the package is imported from, or None when installed."""
+    root = package_dir().parent
+    return root if (root / "pyproject.toml").is_file() else None
+
+
+def home_dir() -> Path:
+    """Directory for runtime data: ``DOCSGPT_HOME``, else the checkout, else cwd."""
+    configured = os.environ.get(HOME_ENV)
+    if configured:
+        return Path(configured).expanduser().resolve()
+    return checkout_root() or Path.cwd()
+
+
+def env_file() -> Path:
+    """The ``.env`` file settings load: ``DOCSGPT_ENV_FILE``, else ``<home>/.env``."""
+    configured = os.environ.get(ENV_FILE_ENV)
+    if configured:
+        path = Path(configured).expanduser()
+        if not path.is_file():
+            raise FileNotFoundError(f"{ENV_FILE_ENV} is set to {path}, which is not a file")
+        return path
+    return home_dir() / ".env"

docsgpt/core/settings.py

@@ -1,17 +1,17 @@
 import os
-from pathlib import Path
 from typing import Optional
 
 from pydantic import AliasChoices, Field, field_validator
 from pydantic_settings import BaseSettings, SettingsConfigDict
 
-current_dir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
-
-
-from docsgpt.core.db_uri import (  # noqa: E402
+from docsgpt.core.db_uri import (
     normalize_pgvector_connection_string,
     normalize_postgres_uri,
 )
+from docsgpt.core.paths import env_file, home_dir
+
+# Runtime data home (DOCSGPT_HOME, the checkout, or cwd); see docsgpt.core.paths.
+current_dir = str(home_dir())
 
 
 class Settings(BaseSettings):
@@ -326,11 +326,12 @@ class Settings(BaseSettings):
     PGVECTOR_IVFFLAT_PROBES: Optional[int] = None
     # Milvus vectorstore config
     MILVUS_COLLECTION_NAME: Optional[str] = "docsgpt"
-    MILVUS_URI: Optional[str] = "./milvus_local.db"  # milvus lite version as default
+    # milvus-lite (embedded) database file, under the data home like the other local stores
+    MILVUS_URI: Optional[str] = Field(default_factory=lambda: str(home_dir() / "milvus_local.db"))
     MILVUS_TOKEN: Optional[str] = ""
 
     # LanceDB vectorstore config
-    LANCEDB_PATH: str = "./data/lancedb"  # Path where LanceDB stores its local data
+    LANCEDB_PATH: str = Field(default_factory=lambda: str(home_dir() / "data" / "lancedb"))  # LanceDB local data
     LANCEDB_TABLE_NAME: Optional[str] = "docsgpts"  # Name of the table to use for storing vectors
 
     FLASK_DEBUG_MODE: bool = False
@@ -581,6 +582,4 @@ class Settings(BaseSettings):
         return stripped
 
 
-# Project root is one level above application/
-path = Path(__file__).parent.parent.parent.absolute()
-settings = Settings(_env_file=path.joinpath(".env"), _env_file_encoding="utf-8")
+settings = Settings(_env_file=env_file(), _env_file_encoding="utf-8")

docsgpt/scripts/prefetch_models.py

@@ -97,11 +97,21 @@ def prefetch(names: Sequence[str], cache_dir: Optional[str] = None) -> List[str]
     return fetched
 
 
+def _parse(argv: Optional[Sequence[str]], prog: str, description: str) -> list[str]:
+    import argparse
+
+    parser = argparse.ArgumentParser(prog=prog, description=description)
+    parser.add_argument(
+        "models", nargs="*", help=f"embedding model names or aliases (default: {', '.join(DEFAULT_MODELS)})"
+    )
+    return parser.parse_args(argv).models or list(DEFAULT_MODELS)
+
+
 def main(argv: Optional[Sequence[str]] = None) -> int:
     logging.basicConfig(level=logging.INFO, format="%(levelname)s %(message)s")
     import os
 
-    names = list(argv) if argv else list(DEFAULT_MODELS)
+    names = _parse(argv, "prefetch-models", "Download the embedding models and the tiktoken encodings into the local caches.")
     fetched = prefetch(names, os.environ.get("EMBEDDINGS_CACHE_DIR"))
     logger.info("Cached %d model(s): %s", len(fetched), ", ".join(fetched))
     encodings = prefetch_tiktoken()

docsgpt/scripts/verify_offline.py

@@ -138,8 +138,16 @@ def _docling_check() -> object:
 
 
 def main(argv: Optional[Sequence[str]] = None) -> int:
+    import argparse
+
     logging.basicConfig(level=logging.WARNING, format="%(levelname)s %(message)s")
-    models: List[str] = list(argv) if argv else list(DEFAULT_MODELS)
+    parser = argparse.ArgumentParser(
+        prog="verify-offline", description="Check that the install serves its defaults without network access."
+    )
+    parser.add_argument(
+        "models", nargs="*", help=f"embedding model names or aliases to check (default: {', '.join(DEFAULT_MODELS)})"
+    )
+    models: List[str] = parser.parse_args(argv).models or list(DEFAULT_MODELS)
     if _network_reachable():
         print("note  network is reachable; run with --network none to prove the offline path")
     else:

docsgpt/storage/local.py

@@ -4,6 +4,7 @@ import shutil
 import tempfile
 from typing import BinaryIO, List, Callable
 
+from docsgpt.core.paths import home_dir
 from docsgpt.storage.base import BaseStorage
 
 
@@ -15,11 +16,10 @@ class LocalStorage(BaseStorage):
         Initialize local storage.
 
         Args:
-            base_dir: Base directory for all operations. If None, uses current directory.
+            base_dir: Base directory for all operations. Defaults to the data home
+                (see ``docsgpt.core.paths.home_dir``).
         """
-        self.base_dir = base_dir or os.path.dirname(
-            os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
-        )
+        self.base_dir = base_dir or str(home_dir())
 
     def _get_full_path(self, path: str) -> str:
         """Get absolute path by combining base_dir and path.

docsgpt/worker.py

@@ -380,10 +380,6 @@ def generate_random_string(length):
     return "".join([string.ascii_letters[i % 52] for i in range(length)])
 
 
-current_dir = os.path.dirname(
-    os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
-)
-
 # Zip extraction security limits. Kept as module constants for backward
 # compatibility with worker callers/tests; values come from operator settings.
 MAX_UNCOMPRESSED_SIZE = settings.UPLOAD_MAX_ARCHIVE_BYTES

pyproject.toml

@@ -1,58 +1,80 @@
+[build-system]
+requires = ["hatchling>=1.27,<2"]
+build-backend = "hatchling.build"
+
 [project]
 name = "docsgpt"
-# Bump together with docsgpt/version.py and frontend/package.json.
-version = "0.19.0"
+# The version is read from docsgpt/version.py (the release workflow reads the
+# same file); bump it there.
+dynamic = ["version"]
 description = "DocsGPT backend: chat with your documents, agents, and tools."
 readme = "README.md"
 requires-python = ">=3.12"
-license = { file = "LICENSE" }
+license = "MIT"
+license-files = ["LICENSE"]
+authors = [{ name = "Arc53" }]
+keywords = ["rag", "llm", "agents", "documents", "chat", "search"]
+classifiers = [
+  "Development Status :: 5 - Production/Stable",
+  "Environment :: Web Environment",
+  "Framework :: Flask",
+  "Intended Audience :: Developers",
+  "Operating System :: OS Independent",
+  "Programming Language :: Python :: 3",
+  "Programming Language :: Python :: 3.12",
+  "Programming Language :: Python :: 3.13",
+  "Topic :: Scientific/Engineering :: Artificial Intelligence",
+]
 
-# Direct dependencies only. Transitive pins live in uv.lock; the pip-facing
-# files under docsgpt/ (requirements*.txt) are exported from that lock by
-# scripts/export_requirements.sh and must not be edited by hand.
+# Direct dependencies only, as compatible ranges: the floor is the version
+# uv.lock pins, the ceiling the next major (next minor below 1.0), so the
+# PyPI package installs next to other packages. The pip-facing files under
+# docsgpt/ (requirements*.txt) are exported from the lock by
+# scripts/export_requirements.sh and must not be edited by hand; Docker, CI
+# and the checkout install those exact versions.
 dependencies = [
-  "a2wsgi==1.10.10",
+  "a2wsgi>=1.10.10,<2",
   "alembic>=1.13,<2",
-  "anthropic==0.121.0",
-  "beautifulsoup4==4.15.0",
-  "boto3==1.43.67",
-  "cel-python==0.5.0",
-  "celery==5.6.3",
-  "celery-redbeat==2.4.2",
-  "croniter==6.2.4",
-  "cryptography==50.0.0",
-  "dataclasses-json==0.6.7",
-  "daytona==0.205.1",
-  "ddgs>=8.0.0",
-  "defusedxml==0.7.1",
-  "docx2txt==0.9",
-  "elevenlabs==2.62.0",
-  "faiss-cpu==1.15.0",
-  "fast-ebook",
+  "anthropic>=0.121.0,<0.122",
+  "beautifulsoup4>=4.15.0,<5",
+  "boto3>=1.43.67,<2",
+  "cel-python>=0.5.0,<0.6",
+  "celery>=5.6.3,<6",
+  "celery-redbeat>=2.4.2,<3",
+  "croniter>=6.2.4,<7",
+  "cryptography>=50.0.0,<51",
+  "dataclasses-json>=0.6.7,<0.7",
+  "daytona>=0.205.1,<0.206",
+  "ddgs>=8.0.0,<10",
+  "defusedxml>=0.7.1,<0.8",
+  "docx2txt>=0.9,<0.10",
+  "elevenlabs>=2.62.0,<3",
+  "faiss-cpu>=1.15.0,<2",
+  "fast-ebook>=0.2.0,<0.3",
   # Default document converter (DOC_PARSER_ENGINE=anydoc): a Rust extension
   # with no model downloads. The docling engine is the `docling` extra.
-  "firecrawl-anydoc==0.2.3",
-  "fastembed==0.8.0",
-  "fastmcp==3.4.6",
-  "Flask==3.1.3",
-  "flask-restx==1.3.2",
-  "google-api-python-client==2.198.0",
-  "google-auth-oauthlib==1.4.0",
-  "google-genai==2.17.0",
-  "gTTS==2.5.4",
-  "gunicorn==26.0.0",
-  "jinja2==3.1.6",
-  "kombu==5.6.2",
-  "markdownify==1.2.3",
-  "msal==1.37.0",
-  "networkx==3.6.1",
-  "numpy==2.5.1",
+  "firecrawl-anydoc>=0.2.3,<0.3",
+  "fastembed>=0.8.0,<0.9",
+  "fastmcp>=3.4.6,<4",
+  "Flask>=3.1.3,<4",
+  "flask-restx>=1.3.2,<2",
+  "google-api-python-client>=2.198.0,<3",
+  "google-auth-oauthlib>=1.4.0,<2",
+  "google-genai>=2.17.0,<3",
+  "gTTS>=2.5.4,<3",
+  "gunicorn>=26.0.0,<27",
+  "jinja2>=3.1.6,<4",
+  "kombu>=5.6.2,<6",
+  "markdownify>=1.2.3,<2",
+  "msal>=1.37.0,<2",
+  "networkx>=3.6.1,<4",
+  "numpy>=2.5.1,<3",
   # fastembed's runtime: local embeddings execute on it.
-  "onnxruntime==1.28.0",
-  "openai==2.53.0",
-  "openapi3-parser==1.1.22",
+  "onnxruntime>=1.28.0,<2",
+  "openai>=2.53.0,<3",
+  "openapi3-parser>=1.1.22,<2",
   # pandas reads .xlsx through openpyxl but does not depend on it.
-  "openpyxl==3.1.5",
+  "openpyxl>=3.1.5,<4",
   "opentelemetry-distro>=0.50b0,<1",
   "opentelemetry-exporter-otlp>=1.29.0,<2",
   "opentelemetry-instrumentation-celery>=0.50b0,<1",
@@ -63,59 +85,72 @@ dependencies = [
   "opentelemetry-instrumentation-requests>=0.50b0,<1",
   "opentelemetry-instrumentation-sqlalchemy>=0.50b0,<1",
   "opentelemetry-instrumentation-starlette>=0.50b0,<1",
-  "pandas==3.0.5",
-  "pdf2image>=1.17.0",
+  "pandas>=3.0.5,<4",
+  "pdf2image>=1.17.0,<2",
   "pgvector>=0.5,<1",
-  "pillow==12.3.0",
-  "praw==8.0.2",
+  "pillow>=12.3.0,<13",
+  "praw>=8.0.2,<9",
   "psycopg[binary,pool]>=3.1,<4",
-  "pydantic",
-  "pydantic-settings",
-  "pypdf==6.15.0",
-  "pypdfium2==5.12.1",
-  "python-dateutil==2.9.0.post0",
-  "python-dotenv",
-  "python-jose==3.5.0",
-  "python-pptx==1.0.2",
-  "PyYAML",
-  "qdrant-client==1.19.0",
-  "redis==7.4.0",
-  "requests==2.34.2",
-  "retry==0.9.2",
+  "pydantic>=2.13.5,<3",
+  "pydantic-settings>=2.15.0,<3",
+  "pypdf>=6.15.0,<7",
+  "pypdfium2>=5.12.1,<6",
+  "python-dateutil>=2.9.0,<3",
+  "python-dotenv>=1.2.3,<2",
+  "python-jose>=3.5.0,<4",
+  "python-pptx>=1.0.2,<2",
+  "PyYAML>=6.0.3,<7",
+  "qdrant-client>=1.19.0,<2",
+  "redis>=7.4.0,<8",
+  "requests>=2.34.2,<3",
+  "retry>=0.9.2,<0.10",
   "sqlalchemy>=2.0,<3",
   "starlette>=1.0,<2",
-  "tiktoken==0.13.0",
-  "tldextract==5.3.2",
-  "tokenizers==0.22.2",
-  "tqdm==4.67.3",
+  "tiktoken>=0.13.0,<0.14",
+  "tldextract>=5.3.2,<6",
+  "tokenizers>=0.22.2,<0.23",
+  "tqdm>=4.67.3,<5",
   "uvicorn[standard]>=0.30,<1",
   "uvicorn-worker>=0.4,<1",
-  "websocket-client==1.9.0",
-  "werkzeug>=3.1.0",
+  "websocket-client>=1.9.0,<2",
+  "werkzeug>=3.1.0,<4",
 ]
 
 [project.optional-dependencies]
 # Docling parser engine: DOC_PARSER_ENGINE=docling, the docling OCR backend
 # (layout-model hybrid OCR, ocrmac/rapidocr engines), .adoc/.vtt/.xml
 # attachment parsing, and read_document's `structured` output. Pulls torch and
-# transformers; on Linux torch resolves from the CPU-only PyTorch index (see
-# [tool.uv.sources]) so the extra does not drag the CUDA stack in.
+# transformers; with uv on Linux torch resolves from the CPU-only PyTorch index
+# (see [tool.uv.sources]) so the extra does not drag the CUDA stack in. pip
+# users on Linux install torch and torchvision from that index first
+# (--index-url https://download.pytorch.org/whl/cpu), then the extra; pip keeps
+# the torch it already has.
 docling = [
-  "docling==2.119.0",
-  "rapidocr==3.9.2",
-  # docling's model stack. Declared here (not left transitive) so the pins
+  "docling>=2.119.0,<3",
+  "rapidocr>=3.9.2,<4",
+  # docling's model stack. Declared here (not left transitive) so the floors
   # hold and the CPU index source below applies. transformers is capped by
   # docling-core at <5.9: 5.9+ breaks the PDF layout model on Apple Silicon.
-  "torch==2.11.0",
-  "torchvision==0.26.0",
-  "transformers==5.8.1",
+  "torch>=2.11.0,<3",
+  "torchvision>=0.26.0,<0.27",
+  "transformers>=5.8.1,<5.9",
 ]
 # VECTOR_STORE=milvus. milvus-lite (the embedded server) pulls pyarrow.
 milvus = [
-  "pymilvus==3.0.1",
-  "milvus-lite==3.2.0; sys_platform != 'win32'",
+  "pymilvus>=3.0.1,<4",
+  "milvus-lite>=3.2.0,<4; sys_platform != 'win32'",
 ]
 
+[project.scripts]
+docsgpt = "docsgpt.cli:main"
+
+[project.urls]
+Homepage = "https://www.docsgpt.cloud/"
+Documentation = "https://docs.docsgpt.cloud/"
+Repository = "https://github.com/arc53/DocsGPT"
+Issues = "https://github.com/arc53/DocsGPT/issues"
+Changelog = "https://github.com/arc53/DocsGPT/releases"
+
 [dependency-groups]
 # Mirrors tests/requirements.txt for `uv sync`; pip users install that file.
 dev = [
@@ -131,10 +166,36 @@ dev = [
   "ruff",
 ]
 
-[tool.uv]
-# The repo is run in place (`application.*` imported from the checkout), not
-# installed as a distribution.
-package = false
+[tool.hatch.version]
+path = "docsgpt/version.py"
+
+# The wheel is the docsgpt package with the data it reads at runtime: prompts,
+# model catalogs, the seed config, alembic.ini and the migrations. Build inputs
+# (Dockerfile, exported requirements), the sample index and local runtime data
+# stay out. The one-release `application` import alias is checkout-only.
+[tool.hatch.build.targets.wheel]
+packages = ["docsgpt"]
+exclude = [
+  "docsgpt/Dockerfile",
+  "docsgpt/requirements*.txt",
+  "docsgpt/index.faiss",
+  "docsgpt/index.pkl",
+  "docsgpt/indexes/",
+  "docsgpt/inputs/",
+  "docsgpt/vectors/",
+]
+
+[tool.hatch.build.targets.sdist]
+include = ["/docsgpt"]
+exclude = [
+  "docsgpt/Dockerfile",
+  "docsgpt/requirements*.txt",
+  "docsgpt/index.faiss",
+  "docsgpt/index.pkl",
+  "docsgpt/indexes/",
+  "docsgpt/inputs/",
+  "docsgpt/vectors/",
+]
 
 [[tool.uv.index]]
 name = "pytorch-cpu"

tests/core/test_paths.py

@@ -0,0 +1,72 @@
+"""Data home and .env discovery (docsgpt.core.paths)."""
+
+from pathlib import Path
+
+import pytest
+
+from docsgpt.core import paths
+
+REPO_ROOT = Path(__file__).resolve().parents[2]
+
+
+class TestHomeDir:
+    def test_the_checkout_wins_over_cwd(self, monkeypatch, tmp_path):
+        monkeypatch.delenv(paths.HOME_ENV, raising=False)
+        monkeypatch.chdir(tmp_path)
+        assert paths.checkout_root() == REPO_ROOT
+        assert paths.home_dir() == REPO_ROOT
+
+    def test_the_env_var_wins(self, monkeypatch, tmp_path):
+        monkeypatch.setenv(paths.HOME_ENV, str(tmp_path))
+        assert paths.home_dir() == tmp_path.resolve()
+
+    def test_an_installed_package_falls_back_to_cwd(self, monkeypatch, tmp_path):
+        monkeypatch.delenv(paths.HOME_ENV, raising=False)
+        monkeypatch.setattr(paths, "checkout_root", lambda: None)
+        monkeypatch.chdir(tmp_path)
+        assert paths.home_dir() == Path.cwd()
+
+
+class TestEnvFile:
+    def test_default_is_dot_env_in_the_home(self, monkeypatch, tmp_path):
+        monkeypatch.delenv(paths.ENV_FILE_ENV, raising=False)
+        monkeypatch.setenv(paths.HOME_ENV, str(tmp_path))
+        assert paths.env_file() == tmp_path.resolve() / ".env"
+
+    def test_the_env_var_wins(self, monkeypatch, tmp_path):
+        custom = tmp_path / "custom.env"
+        custom.write_text("LLM_NAME=x\n")
+        monkeypatch.setenv(paths.ENV_FILE_ENV, str(custom))
+        assert paths.env_file() == custom
+
+    def test_a_missing_configured_file_is_an_error(self, monkeypatch, tmp_path):
+        """Booting with every default because of a typo in the path must not be silent."""
+        monkeypatch.setenv(paths.ENV_FILE_ENV, str(tmp_path / "nope.env"))
+        with pytest.raises(FileNotFoundError, match="DOCSGPT_ENV_FILE"):
+            paths.env_file()
+
+
+class TestSettingsFollowTheHome:
+    def test_embedded_vector_stores_default_under_the_home(self, monkeypatch, tmp_path):
+        from docsgpt.core.settings import Settings
+
+        monkeypatch.setenv(paths.HOME_ENV, str(tmp_path))
+        monkeypatch.delenv("MILVUS_URI", raising=False)
+        monkeypatch.delenv("LANCEDB_PATH", raising=False)
+        fresh = Settings(_env_file=None)
+        assert fresh.MILVUS_URI == str(tmp_path.resolve() / "milvus_local.db")
+        assert fresh.LANCEDB_PATH == str(tmp_path.resolve() / "data" / "lancedb")
+
+    def test_env_overrides_still_win(self, monkeypatch, tmp_path):
+        from docsgpt.core.settings import Settings
+
+        monkeypatch.setenv(paths.HOME_ENV, str(tmp_path))
+        monkeypatch.setenv("LANCEDB_PATH", "/srv/lancedb")
+        assert Settings(_env_file=None).LANCEDB_PATH == "/srv/lancedb"
+
+
+class TestPackageDir:
+    def test_holds_the_shipped_data(self):
+        assert paths.package_dir().name == "docsgpt"
+        assert (paths.package_dir() / "alembic.ini").is_file()
+        assert (paths.package_dir() / "prompts" / "chat_reduce_prompt.txt").is_file()

tests/test_cli.py

@@ -0,0 +1,151 @@
+"""The ``docsgpt`` command dispatches to the API server, the worker and the scripts."""
+
+import subprocess
+import sys
+import types
+from pathlib import Path
+from unittest.mock import MagicMock
+
+import click
+import pytest
+
+from docsgpt import cli
+from docsgpt.version import __version__
+
+
+class TestTopLevel:
+    def test_version(self, capsys):
+        with pytest.raises(SystemExit) as exc:
+            cli.main(["--version"])
+        assert exc.value.code == 0
+        assert capsys.readouterr().out.strip() == f"docsgpt {__version__}"
+
+    def test_no_command_prints_help(self, capsys):
+        assert cli.main([]) == 2
+        assert "worker" in capsys.readouterr().out
+
+    def test_importing_the_cli_does_not_boot_the_app(self):
+        """``docsgpt --help`` must not import the Flask app, Celery or settings."""
+        code = (
+            "import sys, docsgpt.cli; "
+            "loaded = {m for m in sys.modules if m in ('docsgpt.app', 'docsgpt.core.settings', 'celery', 'flask')}; "
+            "assert not loaded, loaded"
+        )
+        subprocess.run([sys.executable, "-c", code], cwd=Path(__file__).resolve().parents[1], check=True)
+
+
+class TestApi:
+    def test_gunicorn_argv(self, monkeypatch, capsys):
+        run = MagicMock()
+        monkeypatch.setitem(sys.modules, "gunicorn.app.wsgiapp", types.SimpleNamespace(run=run))
+        monkeypatch.setattr(sys, "platform", "linux")
+        monkeypatch.setattr(sys, "argv", ["docsgpt"])
+        assert cli.main(["api", "--port", "8000", "--workers", "2"]) == 0
+        run.assert_called_once_with()
+        argv = sys.argv
+        assert argv[0] == "gunicorn" and argv[-1] == "docsgpt.asgi:asgi_app"
+        assert argv[argv.index("--bind") + 1] == "127.0.0.1:8000"
+        assert argv[argv.index("-w") + 1] == "2"
+        assert argv[argv.index("-k") + 1] == "docsgpt.gunicorn_worker.BoundedDrainUvicornWorker"
+        assert argv[argv.index("--config") + 1] == "python:docsgpt.gunicorn_conf"
+        assert "data home" in capsys.readouterr().err
+
+    def test_reload_uses_uvicorn(self, monkeypatch):
+        uvicorn = types.SimpleNamespace(run=MagicMock())
+        monkeypatch.setitem(sys.modules, "uvicorn", uvicorn)
+        assert cli.main(["api", "--reload", "--host", "127.0.0.1"]) == 0
+        uvicorn.run.assert_called_once_with("docsgpt.asgi:asgi_app", host="127.0.0.1", port=7091, reload=True)
+
+
+class TestWorker:
+    @staticmethod
+    def _celery(monkeypatch, start=None):
+        celery = MagicMock()
+        celery.start = start or MagicMock(return_value=0)
+        monkeypatch.setitem(sys.modules, "docsgpt.app", types.SimpleNamespace(celery=celery))
+        return celery
+
+    def test_defaults_consume_every_configured_queue(self, monkeypatch, capsys):
+        celery = self._celery(monkeypatch)
+        monkeypatch.setattr(sys, "platform", "linux")
+        assert cli.main(["worker"]) == 0
+        argv = celery.start.call_args.args[0]
+        assert argv[:3] == ["worker", "-l", "INFO"]
+        assert "-Q" not in argv, "a bare worker honours EMBEDDINGS_QUEUE and DOCUMENT_PARSE_QUEUE"
+        assert "-B" in argv
+        assert "--pool" not in argv
+        assert "data home" in capsys.readouterr().err
+
+    def test_options_and_the_macos_solo_pool(self, monkeypatch):
+        celery = self._celery(monkeypatch)
+        monkeypatch.setattr(sys, "platform", "darwin")
+        assert cli.main(["worker", "--no-beat", "-Q", "embeddings", "--concurrency", "2"]) == 0
+        argv = celery.start.call_args.args[0]
+        assert argv[argv.index("--pool") + 1] == "solo"
+        assert "-B" not in argv
+        assert argv[argv.index("-Q") + 1] == "embeddings"
+        assert argv[argv.index("--concurrency") + 1] == "2"
+
+    def test_windows_runs_solo_without_the_embedded_scheduler(self, monkeypatch, capsys):
+        celery = self._celery(monkeypatch)
+        monkeypatch.setattr(sys, "platform", "win32")
+        assert cli.main(["worker"]) == 0
+        argv = celery.start.call_args.args[0]
+        assert "-B" not in argv
+        assert argv[argv.index("--pool") + 1] == "solo"
+        assert "docsgpt beat" in capsys.readouterr().err
+
+    def test_the_worker_exit_code_is_returned(self, monkeypatch):
+        self._celery(monkeypatch, start=MagicMock(return_value=1))
+        monkeypatch.setattr(sys, "platform", "linux")
+        assert cli.main(["worker"]) == 1
+
+    def test_a_usage_error_prints_usage_instead_of_a_traceback(self, monkeypatch, capsys):
+        self._celery(monkeypatch, start=MagicMock(side_effect=click.UsageError("No such option: --bogus")))
+        monkeypatch.setattr(sys, "platform", "linux")
+        assert cli.main(["worker"]) == 2
+        assert "No such option" in capsys.readouterr().err
+
+
+class TestBeat:
+    def test_runs_the_scheduler_alone(self, monkeypatch):
+        celery = MagicMock()
+        celery.start = MagicMock(return_value=0)
+        monkeypatch.setitem(sys.modules, "docsgpt.app", types.SimpleNamespace(celery=celery))
+        assert cli.main(["beat", "-l", "DEBUG"]) == 0
+        assert celery.start.call_args.args[0] == ["beat", "-l", "DEBUG"]
+
+
+class TestMigrate:
+    def test_runs_the_bootstrap(self, monkeypatch):
+        ensure = MagicMock()
+        monkeypatch.setattr("docsgpt.storage.db.bootstrap.ensure_database_ready", ensure)
+        monkeypatch.setattr("docsgpt.core.settings.settings.POSTGRES_URI", "postgresql://docsgpt@localhost/docsgpt")
+        assert cli.main(["migrate", "--no-create"]) == 0
+        assert ensure.call_args.args[0] == "postgresql://docsgpt@localhost/docsgpt"
+        assert ensure.call_args.kwargs["create_db"] is False
+        assert ensure.call_args.kwargs["migrate"] is True
+
+    def test_without_a_database_uri(self, monkeypatch, capsys):
+        monkeypatch.setattr("docsgpt.core.settings.settings.POSTGRES_URI", None)
+        assert cli.main(["migrate"]) == 2
+        assert "POSTGRES_URI" in capsys.readouterr().err
+
+
+class TestScripts:
+    def test_arguments_pass_through_untouched(self, monkeypatch):
+        main = MagicMock(return_value=0)
+        monkeypatch.setattr("docsgpt.scripts.prefetch_models.main", main)
+        assert cli.main(["prefetch-models", "--embeddings", "x", "--help"]) == 0
+        main.assert_called_once_with(["--embeddings", "x", "--help"])
+
+    def test_the_script_exit_code_is_returned(self, monkeypatch):
+        monkeypatch.setattr("docsgpt.scripts.verify_offline.main", MagicMock(return_value=3))
+        assert cli.main(["verify-offline"]) == 3
+
+    @pytest.mark.parametrize("script", ["prefetch-models", "verify-offline"])
+    def test_help_is_help_not_a_model_name(self, script, capsys):
+        with pytest.raises(SystemExit) as exc:
+            cli.main([script, "--help"])
+        assert exc.value.code == 0
+        assert "models" in capsys.readouterr().out

uv.lock

@@ -1385,8 +1385,7 @@ standard = [
 
 [[package]]
 name = "docsgpt"
-version = "0.19.0"
-source = { virtual = "." }
+source = { editable = "." }
 dependencies = [
     { name = "a2wsgi" },
     { name = "alembic" },
@@ -1498,46 +1497,46 @@ dev = [
 
 [package.metadata]
 requires-dist = [
-    { name = "a2wsgi", specifier = "==1.10.10" },
+    { name = "a2wsgi", specifier = ">=1.10.10,<2" },
     { name = "alembic", specifier = ">=1.13,<2" },
-    { name = "anthropic", specifier = "==0.121.0" },
-    { name = "beautifulsoup4", specifier = "==4.15.0" },
-    { name = "boto3", specifier = "==1.43.67" },
-    { name = "cel-python", specifier = "==0.5.0" },
-    { name = "celery", specifier = "==5.6.3" },
-    { name = "celery-redbeat", specifier = "==2.4.2" },
-    { name = "croniter", specifier = "==6.2.4" },
-    { name = "cryptography", specifier = "==50.0.0" },
-    { name = "dataclasses-json", specifier = "==0.6.7" },
-    { name = "daytona", specifier = "==0.205.1" },
-    { name = "ddgs", specifier = ">=8.0.0" },
-    { name = "defusedxml", specifier = "==0.7.1" },
-    { name = "docling", marker = "extra == 'docling'", specifier = "==2.119.0" },
-    { name = "docx2txt", specifier = "==0.9" },
-    { name = "elevenlabs", specifier = "==2.62.0" },
-    { name = "faiss-cpu", specifier = "==1.15.0" },
-    { name = "fast-ebook" },
-    { name = "fastembed", specifier = "==0.8.0" },
-    { name = "fastmcp", specifier = "==3.4.6" },
-    { name = "firecrawl-anydoc", specifier = "==0.2.3" },
-    { name = "flask", specifier = "==3.1.3" },
-    { name = "flask-restx", specifier = "==1.3.2" },
-    { name = "google-api-python-client", specifier = "==2.198.0" },
-    { name = "google-auth-oauthlib", specifier = "==1.4.0" },
-    { name = "google-genai", specifier = "==2.17.0" },
-    { name = "gtts", specifier = "==2.5.4" },
-    { name = "gunicorn", specifier = "==26.0.0" },
-    { name = "jinja2", specifier = "==3.1.6" },
-    { name = "kombu", specifier = "==5.6.2" },
-    { name = "markdownify", specifier = "==1.2.3" },
-    { name = "milvus-lite", marker = "sys_platform != 'win32' and extra == 'milvus'", specifier = "==3.2.0" },
-    { name = "msal", specifier = "==1.37.0" },
-    { name = "networkx", specifier = "==3.6.1" },
-    { name = "numpy", specifier = "==2.5.1" },
-    { name = "onnxruntime", specifier = "==1.28.0" },
-    { name = "openai", specifier = "==2.53.0" },
-    { name = "openapi3-parser", specifier = "==1.1.22" },
-    { name = "openpyxl", specifier = "==3.1.5" },
+    { name = "anthropic", specifier = ">=0.121.0,<0.122" },
+    { name = "beautifulsoup4", specifier = ">=4.15.0,<5" },
+    { name = "boto3", specifier = ">=1.43.67,<2" },
+    { name = "cel-python", specifier = ">=0.5.0,<0.6" },
+    { name = "celery", specifier = ">=5.6.3,<6" },
+    { name = "celery-redbeat", specifier = ">=2.4.2,<3" },
+    { name = "croniter", specifier = ">=6.2.4,<7" },
+    { name = "cryptography", specifier = ">=50.0.0,<51" },
+    { name = "dataclasses-json", specifier = ">=0.6.7,<0.7" },
+    { name = "daytona", specifier = ">=0.205.1,<0.206" },
+    { name = "ddgs", specifier = ">=8.0.0,<10" },
+    { name = "defusedxml", specifier = ">=0.7.1,<0.8" },
+    { name = "docling", marker = "extra == 'docling'", specifier = ">=2.119.0,<3" },
+    { name = "docx2txt", specifier = ">=0.9,<0.10" },
+    { name = "elevenlabs", specifier = ">=2.62.0,<3" },
+    { name = "faiss-cpu", specifier = ">=1.15.0,<2" },
+    { name = "fast-ebook", specifier = ">=0.2.0,<0.3" },
+    { name = "fastembed", specifier = ">=0.8.0,<0.9" },
+    { name = "fastmcp", specifier = ">=3.4.6,<4" },
+    { name = "firecrawl-anydoc", specifier = ">=0.2.3,<0.3" },
+    { name = "flask", specifier = ">=3.1.3,<4" },
+    { name = "flask-restx", specifier = ">=1.3.2,<2" },
+    { name = "google-api-python-client", specifier = ">=2.198.0,<3" },
+    { name = "google-auth-oauthlib", specifier = ">=1.4.0,<2" },
+    { name = "google-genai", specifier = ">=2.17.0,<3" },
+    { name = "gtts", specifier = ">=2.5.4,<3" },
+    { name = "gunicorn", specifier = ">=26.0.0,<27" },
+    { name = "jinja2", specifier = ">=3.1.6,<4" },
+    { name = "kombu", specifier = ">=5.6.2,<6" },
+    { name = "markdownify", specifier = ">=1.2.3,<2" },
+    { name = "milvus-lite", marker = "sys_platform != 'win32' and extra == 'milvus'", specifier = ">=3.2.0,<4" },
+    { name = "msal", specifier = ">=1.37.0,<2" },
+    { name = "networkx", specifier = ">=3.6.1,<4" },
+    { name = "numpy", specifier = ">=2.5.1,<3" },
+    { name = "onnxruntime", specifier = ">=1.28.0,<2" },
+    { name = "openai", specifier = ">=2.53.0,<3" },
+    { name = "openapi3-parser", specifier = ">=1.1.22,<2" },
+    { name = "openpyxl", specifier = ">=3.1.5,<4" },
     { name = "opentelemetry-distro", specifier = ">=0.50b0,<1" },
     { name = "opentelemetry-exporter-otlp", specifier = ">=1.29.0,<2" },
     { name = "opentelemetry-instrumentation-celery", specifier = ">=0.50b0,<1" },
@@ -1548,42 +1547,42 @@ requires-dist = [
     { name = "opentelemetry-instrumentation-requests", specifier = ">=0.50b0,<1" },
     { name = "opentelemetry-instrumentation-sqlalchemy", specifier = ">=0.50b0,<1" },
     { name = "opentelemetry-instrumentation-starlette", specifier = ">=0.50b0,<1" },
-    { name = "pandas", specifier = "==3.0.5" },
-    { name = "pdf2image", specifier = ">=1.17.0" },
+    { name = "pandas", specifier = ">=3.0.5,<4" },
+    { name = "pdf2image", specifier = ">=1.17.0,<2" },
     { name = "pgvector", specifier = ">=0.5,<1" },
-    { name = "pillow", specifier = "==12.3.0" },
-    { name = "praw", specifier = "==8.0.2" },
+    { name = "pillow", specifier = ">=12.3.0,<13" },
+    { name = "praw", specifier = ">=8.0.2,<9" },
     { name = "psycopg", extras = ["binary", "pool"], specifier = ">=3.1,<4" },
-    { name = "pydantic" },
-    { name = "pydantic-settings" },
-    { name = "pymilvus", marker = "extra == 'milvus'", specifier = "==3.0.1" },
-    { name = "pypdf", specifier = "==6.15.0" },
-    { name = "pypdfium2", specifier = "==5.12.1" },
-    { name = "python-dateutil", specifier = "==2.9.0.post0" },
-    { name = "python-dotenv" },
-    { name = "python-jose", specifier = "==3.5.0" },
-    { name = "python-pptx", specifier = "==1.0.2" },
-    { name = "pyyaml" },
-    { name = "qdrant-client", specifier = "==1.19.0" },
-    { name = "rapidocr", marker = "extra == 'docling'", specifier = "==3.9.2" },
-    { name = "redis", specifier = "==7.4.0" },
-    { name = "requests", specifier = "==2.34.2" },
-    { name = "retry", specifier = "==0.9.2" },
+    { name = "pydantic", specifier = ">=2.13.5,<3" },
+    { name = "pydantic-settings", specifier = ">=2.15.0,<3" },
+    { name = "pymilvus", marker = "extra == 'milvus'", specifier = ">=3.0.1,<4" },
+    { name = "pypdf", specifier = ">=6.15.0,<7" },
+    { name = "pypdfium2", specifier = ">=5.12.1,<6" },
+    { name = "python-dateutil", specifier = ">=2.9.0,<3" },
+    { name = "python-dotenv", specifier = ">=1.2.3,<2" },
+    { name = "python-jose", specifier = ">=3.5.0,<4" },
+    { name = "python-pptx", specifier = ">=1.0.2,<2" },
+    { name = "pyyaml", specifier = ">=6.0.3,<7" },
+    { name = "qdrant-client", specifier = ">=1.19.0,<2" },
+    { name = "rapidocr", marker = "extra == 'docling'", specifier = ">=3.9.2,<4" },
+    { name = "redis", specifier = ">=7.4.0,<8" },
+    { name = "requests", specifier = ">=2.34.2,<3" },
+    { name = "retry", specifier = ">=0.9.2,<0.10" },
     { name = "sqlalchemy", specifier = ">=2.0,<3" },
     { name = "starlette", specifier = ">=1.0,<2" },
-    { name = "tiktoken", specifier = "==0.13.0" },
-    { name = "tldextract", specifier = "==5.3.2" },
-    { name = "tokenizers", specifier = "==0.22.2" },
-    { name = "torch", marker = "sys_platform == 'linux' and extra == 'docling'", specifier = "==2.11.0", index = "https://download.pytorch.org/whl/cpu" },
-    { name = "torch", marker = "sys_platform != 'linux' and extra == 'docling'", specifier = "==2.11.0" },
-    { name = "torchvision", marker = "sys_platform == 'linux' and extra == 'docling'", specifier = "==0.26.0", index = "https://download.pytorch.org/whl/cpu" },
-    { name = "torchvision", marker = "sys_platform != 'linux' and extra == 'docling'", specifier = "==0.26.0" },
-    { name = "tqdm", specifier = "==4.67.3" },
-    { name = "transformers", marker = "extra == 'docling'", specifier = "==5.8.1" },
+    { name = "tiktoken", specifier = ">=0.13.0,<0.14" },
+    { name = "tldextract", specifier = ">=5.3.2,<6" },
+    { name = "tokenizers", specifier = ">=0.22.2,<0.23" },
+    { name = "torch", marker = "sys_platform == 'linux' and extra == 'docling'", specifier = ">=2.11.0,<3", index = "https://download.pytorch.org/whl/cpu" },
+    { name = "torch", marker = "sys_platform != 'linux' and extra == 'docling'", specifier = ">=2.11.0,<3" },
+    { name = "torchvision", marker = "sys_platform == 'linux' and extra == 'docling'", specifier = ">=0.26.0,<0.27", index = "https://download.pytorch.org/whl/cpu" },
+    { name = "torchvision", marker = "sys_platform != 'linux' and extra == 'docling'", specifier = ">=0.26.0,<0.27" },
+    { name = "tqdm", specifier = ">=4.67.3,<5" },
+    { name = "transformers", marker = "extra == 'docling'", specifier = ">=5.8.1,<5.9" },
     { name = "uvicorn", extras = ["standard"], specifier = ">=0.30,<1" },
     { name = "uvicorn-worker", specifier = ">=0.4,<1" },
-    { name = "websocket-client", specifier = "==1.9.0" },
-    { name = "werkzeug", specifier = ">=3.1.0" },
+    { name = "websocket-client", specifier = ">=1.9.0,<2" },
+    { name = "werkzeug", specifier = ">=3.1.0,<4" },
 ]
 provides-extras = ["docling", "milvus"]