From bea26336f8d9c0989a718103fd9e7b68c153d73f Mon Sep 17 00:00:00 2001 From: Alex Date: Mon, 7 Sep 2026 14:40:35 +0100 Subject: [PATCH 1/4] feat: publish the backend to PyPI as docsgpt pip install docsgpt (extras: docling, milvus) installs the backend with a docsgpt command: api, worker, migrate, prefetch-models, verify-offline, reembed. Second step of the PyPI work after the package rename. - hatchling build; the version comes from docsgpt/version.py. The wheel is the docsgpt package with the data it reads at runtime (prompts, model catalogs, seed config, alembic.ini and migrations) and without the Dockerfile, the exported requirements, the sample index and local runtime data. The application import alias stays checkout-only. uv sync installs the package editable now that [tool.uv] package = false is gone. - docsgpt/cli.py: api (gunicorn + BoundedDrainUvicornWorker with the image's flags, --reload for uvicorn), worker (Celery worker with beat embedded, --no-beat/-Q/--concurrency/--pool, solo pool on macOS), migrate, and argument pass-through to the maintenance scripts. --help imports no app. - docsgpt/core/paths.py: runtime data lives in a data home (DOCSGPT_HOME, else the checkout, else cwd); DOCSGPT_ENV_FILE overrides the env file. Settings, the dotenv load, LocalStorage and the internal upload route use it instead of "three directories above this file", which is site-packages for an installed package. A checkout and the Docker image behave as before. - [project] dependencies are compatible ranges so the package installs next to other packages; uv.lock resolves to the same versions and the exported requirements files are unchanged. - package-build.yml builds and checks the wheel on PRs and installs it into a clean venv; pypi-publish.yml publishes on a published release through trusted publishing (environment pypi), or to TestPyPI on a manual run. - Docs: Deploying -> Install with pip. AGENTS.md notes the package. --- .github/workflows/package-build.yml | 88 ++++++++++ .github/workflows/pypi-publish.yml | 86 ++++++++++ .gitignore | 3 + AGENTS.md | 7 + docs/content/Deploying/Pip-Install.mdx | 86 ++++++++++ docs/content/Deploying/_meta.js | 4 + docsgpt/api/internal/routes.py | 5 +- docsgpt/api/user/base.py | 4 - docsgpt/app.py | 3 +- docsgpt/cli.py | 145 ++++++++++++++++ docsgpt/core/paths.py | 47 ++++++ docsgpt/core/settings.py | 14 +- docsgpt/storage/local.py | 8 +- docsgpt/worker.py | 4 - pyproject.toml | 220 ++++++++++++++++--------- tests/core/test_paths.py | 43 +++++ tests/test_cli.py | 111 +++++++++++++ uv.lock | 141 ++++++++-------- 18 files changed, 844 insertions(+), 175 deletions(-) create mode 100644 .github/workflows/package-build.yml create mode 100644 .github/workflows/pypi-publish.yml create mode 100644 docs/content/Deploying/Pip-Install.mdx create mode 100644 docsgpt/cli.py create mode 100644 docsgpt/core/paths.py create mode 100644 tests/core/test_paths.py create mode 100644 tests/test_cli.py diff --git a/.github/workflows/package-build.yml b/.github/workflows/package-build.yml new file mode 100644 index 00000000..f6cd3296 --- /dev/null +++ b/.github/workflows/package-build.yml @@ -0,0 +1,88 @@ +name: Build the Python package + +# Builds the sdist and wheel, checks the wheel carries the data the app reads +# at runtime and nothing from the checkout, then installs it into a clean +# environment and runs the docsgpt command. + +on: + pull_request: + paths: + - 'pyproject.toml' + - 'uv.lock' + - 'docsgpt/**' + - 'README.md' + - 'LICENSE' + - '.github/workflows/package-build.yml' + push: + branches: [main] + paths: + - 'pyproject.toml' + - 'uv.lock' + - 'docsgpt/**' + - 'README.md' + - 'LICENSE' + - '.github/workflows/package-build.yml' + workflow_dispatch: + +permissions: + contents: read + +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + with: + persist-credentials: false + + - uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6.8.0 + + - name: Build the sdist and wheel + run: uv build + + - name: Check the archives + run: | + uvx twine check --strict dist/* + python3 - <<'PY' + import glob + import zipfile + + names = set(zipfile.ZipFile(glob.glob("dist/*.whl")[0]).namelist()) + for required in ( + "docsgpt/cli.py", + "docsgpt/alembic.ini", + "docsgpt/alembic/env.py", + "docsgpt/alembic/script.py.mako", + "docsgpt/prompts/chat_reduce_prompt.txt", + "docsgpt/prompts/fragments/identity.txt", + "docsgpt/core/models/_defaults.yaml", + "docsgpt/seed/config/premade_agents.yaml", + ): + assert required in names, f"missing from the wheel: {required}" + for banned in ( + "docsgpt/Dockerfile", + "docsgpt/requirements.txt", + "docsgpt/requirements-docling.txt", + "docsgpt/index.faiss", + "docsgpt/index.pkl", + "application/__init__.py", + ): + assert banned not in names, f"must not ship: {banned}" + stray = [n for n in names if n.startswith(("docsgpt/indexes/", "docsgpt/inputs/", "docsgpt/vectors/"))] + assert not stray, stray + print(f"wheel ok: {len(names)} files") + PY + + - name: Install the wheel into a clean environment and run the command + run: | + uv venv "$RUNNER_TEMP/docsgpt-venv" + uv pip install --python "$RUNNER_TEMP/docsgpt-venv/bin/python" dist/*.whl + "$RUNNER_TEMP/docsgpt-venv/bin/docsgpt" --version + "$RUNNER_TEMP/docsgpt-venv/bin/docsgpt" --help + "$RUNNER_TEMP/docsgpt-venv/bin/docsgpt" api --help + "$RUNNER_TEMP/docsgpt-venv/bin/python" -c "from docsgpt.core import paths; print('home:', paths.home_dir())" + + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: dist + path: dist/ diff --git a/.github/workflows/pypi-publish.yml b/.github/workflows/pypi-publish.yml new file mode 100644 index 00000000..8e977c56 --- /dev/null +++ b/.github/workflows/pypi-publish.yml @@ -0,0 +1,86 @@ +name: Publish to PyPI + +# Trusted publishing: PyPI trusts this workflow file in arc53/DocsGPT running +# in the `pypi` environment, so no API token is stored. A published release +# (which the backend-release workflow creates when docsgpt/version.py changes +# on main) publishes to PyPI; a manual run publishes to TestPyPI by default. + +on: + release: + types: [published] + workflow_dispatch: + inputs: + target: + description: Package index to publish to + type: choice + options: [testpypi, pypi] + default: testpypi + +permissions: + contents: read + +jobs: + build: + if: github.repository == 'arc53/DocsGPT' + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + with: + persist-credentials: false + + - uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6.8.0 + with: + # A publishing workflow must not restore a cache another job could have poisoned. + enable-cache: false + + - name: Build the sdist and wheel + run: uv build + + - name: The archives carry the released version + if: github.event_name == 'release' + env: + TAG: ${{ github.event.release.tag_name }} + run: | + ls "dist/docsgpt-${TAG}.tar.gz" "dist/docsgpt-${TAG}-py3-none-any.whl" + + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: dist + path: dist/ + if-no-files-found: error + + publish-pypi: + if: github.event_name == 'release' || inputs.target == 'pypi' + needs: build + runs-on: ubuntu-latest + environment: + name: pypi + url: https://pypi.org/project/docsgpt/ + permissions: + id-token: write + steps: + - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: dist + path: dist/ + + - uses: pypa/gh-action-pypi-publish@dc37677b2e1c63e2034f94d8a5b11f265b73ba33 # v1.14.2 + + publish-testpypi: + if: github.event_name == 'workflow_dispatch' && inputs.target == 'testpypi' + needs: build + runs-on: ubuntu-latest + environment: + name: testpypi + url: https://test.pypi.org/project/docsgpt/ + permissions: + id-token: write + steps: + - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: dist + path: dist/ + + - uses: pypa/gh-action-pypi-publish@dc37677b2e1c63e2034f94d8a5b11f265b73ba33 # v1.14.2 + with: + repository-url: https://test.pypi.org/legacy/ diff --git a/.gitignore b/.gitignore index d2e4019f..8693a33e 100644 --- a/.gitignore +++ b/.gitignore @@ -206,3 +206,6 @@ tests/e2e/.e2e-last-run.json docsgpt/core/models/foundry_test_internal.yaml bench/ /benchmark/ + +# Python package build output (uv build) +dist/ diff --git a/AGENTS.md b/AGENTS.md index eb4d9708..8b878d56 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -37,6 +37,13 @@ uv pip install -r docsgpt/requirements.txt # or: pip install -r docsgpt/require # (the file adds the PyTorch CPU index; prefer `uv sync --extra docling`). ``` +The backend is also an installable package (`pyproject.toml`, hatchling). +`uv sync` installs it editable and puts a `docsgpt` command on PATH: +`docsgpt api --reload`, `docsgpt worker`, `docsgpt migrate`, +`docsgpt prefetch-models`, `docsgpt verify-offline`. Runtime data (`.env`, +`inputs/`, `indexes/`) lives in the checkout by default; `DOCSGPT_HOME` and +`DOCSGPT_ENV_FILE` move it (see `docsgpt/core/paths.py`). + Dependencies are declared in `pyproject.toml` and locked in `uv.lock`; the `docsgpt/requirements*.txt` files are exported from the lock. To add or bump a package: edit `pyproject.toml`, run `uv lock`, then diff --git a/docs/content/Deploying/Pip-Install.mdx b/docs/content/Deploying/Pip-Install.mdx new file mode 100644 index 00000000..41586200 --- /dev/null +++ b/docs/content/Deploying/Pip-Install.mdx @@ -0,0 +1,86 @@ +--- +title: Install with pip +description: Run the DocsGPT backend from the PyPI package, inside your own Python environment. +--- + +import { Callout } from 'nextra/components' + +# Install with pip + +The DocsGPT backend is on PyPI as [`docsgpt`](https://pypi.org/project/docsgpt/): the API server, the Celery worker and the maintenance scripts in one package, behind a single `docsgpt` command. Use it when you want DocsGPT inside your own Python environment or process manager rather than the [Docker images](/Deploying/Docker-Deploying). + + + The package is the backend only. The web UI ships in the Docker images and in the repository under `frontend/`; serving it from the package is planned. + + +## Requirements + +- Python 3.12 or newer +- [PostgreSQL](/Deploying/Postgres-Migration) for user data, with the `vector` extension if you set `VECTOR_STORE=pgvector` +- Redis for the task queue and the cache +- An LLM: an API key for a hosted provider, or a local model server + +## Install + +```bash +python -m venv .venv && source .venv/bin/activate +pip install docsgpt +``` + +Extras add the optional engines: + +```bash +pip install "docsgpt[docling]" # DOC_PARSER_ENGINE=docling: OCR backend, structured output +pip install "docsgpt[milvus]" # VECTOR_STORE=milvus +``` + +The `docling` extra pulls in PyTorch. On Linux, PyPI's torch wheels bring the CUDA stack with them; on a CPU-only machine add `--extra-index-url https://download.pytorch.org/whl/cpu` to the install command to get the CPU build instead. `uv pip install` and `pipx install` work the same way. + +## Configure + +DocsGPT keeps its runtime files in a **data home**: the `.env` file it reads settings from, uploaded files under `inputs/` and vector indexes under `indexes/`. The data home is the directory you run the commands from, or the directory `DOCSGPT_HOME` points to. `DOCSGPT_ENV_FILE` points at a `.env` kept somewhere else. + +Create a `.env` in the data home. The minimum for a hosted LLM: + +```ini +# LLM provider and model: see App Configuration for the options +LLM_PROVIDER=openai +LLM_NAME= +API_KEY= + +# User data +POSTGRES_URI=postgresql://docsgpt:docsgpt@localhost:5432/docsgpt + +# Worker-to-API authentication (required for uploads) +INTERNAL_KEY= +``` + +Redis defaults to `localhost:6379` (`CELERY_BROKER_URL`, `CELERY_RESULT_BACKEND`, `CACHE_REDIS_URL`). Every setting is listed in [App Configuration](/Deploying/DocsGPT-Settings). + +## Run + +```bash +docsgpt migrate # create the database if it is missing and apply the migrations +docsgpt api # the API on http://localhost:7091 +docsgpt worker # in a second terminal: the Celery worker, with the scheduler +``` + +The API applies pending migrations when it starts (`AUTO_MIGRATE`), so `docsgpt migrate` is the explicit step for deployments that want the schema in place before the first request or that run the API with a restricted database role. + +The worker is not optional: query embedding runs on it, so search fails without one. `docsgpt worker --help` lists the queue, concurrency and pool options; `--no-beat` starts a worker without the scheduler when another worker already runs it. + +Other commands: + +- `docsgpt api --reload`: a development server with auto-reload. +- `docsgpt prefetch-models`: download the embedding, tokenizer and parser models ahead of time, for machines that go offline. +- `docsgpt verify-offline`: check that a prepared install starts with networking off. +- `docsgpt reembed`: re-embed every index after changing `EMBEDDINGS_NAME` (see [Upgrading](/upgrading)). + +## Upgrade + +```bash +pip install -U docsgpt +docsgpt migrate +``` + +Read the [upgrade notes](/upgrading) first when moving between minor versions. diff --git a/docs/content/Deploying/_meta.js b/docs/content/Deploying/_meta.js index 5f414192..c14234bd 100644 --- a/docs/content/Deploying/_meta.js +++ b/docs/content/Deploying/_meta.js @@ -15,6 +15,10 @@ export default { "title": "🛳️ Docker Setup", "href": "/Deploying/Docker-Deploying" }, + "Pip-Install": { + "title": "🐍 Install with pip", + "href": "/Deploying/Pip-Install" + }, "Development-Environment": { "title": "🛠️Development Environment", "href": "/Deploying/Development-Environment" diff --git a/docsgpt/api/internal/routes.py b/docsgpt/api/internal/routes.py index 6c097207..d849c77f 100755 --- a/docsgpt/api/internal/routes.py +++ b/docsgpt/api/internal/routes.py @@ -5,6 +5,7 @@ from flask import Blueprint, request, send_from_directory, jsonify from werkzeug.utils import secure_filename import logging +from docsgpt.core.paths import home_dir from docsgpt.core.settings import settings from docsgpt.storage.db.base_repository import looks_like_uuid from docsgpt.storage.db.repositories.sources import SourcesRepository @@ -14,9 +15,7 @@ from docsgpt.storage.storage_creator import StorageCreator logger = logging.getLogger(__name__) -current_dir = os.path.dirname( - os.path.dirname(os.path.dirname(os.path.abspath(__file__))) -) +current_dir = str(home_dir()) internal = Blueprint("internal", __name__) diff --git a/docsgpt/api/user/base.py b/docsgpt/api/user/base.py index 8cd85b8a..f6cd6f58 100644 --- a/docsgpt/api/user/base.py +++ b/docsgpt/api/user/base.py @@ -32,10 +32,6 @@ from docsgpt.vectorstore.vector_creator import VectorCreator storage = StorageCreator.get_storage() -current_dir = os.path.dirname( - os.path.dirname(os.path.dirname(os.path.abspath(__file__))) -) - def generate_minute_range(start_date, end_date): """Generate a dictionary with minute-level time ranges.""" return { diff --git a/docsgpt/app.py b/docsgpt/app.py index 96b392aa..81c303dc 100644 --- a/docsgpt/app.py +++ b/docsgpt/app.py @@ -11,6 +11,7 @@ from werkzeug.exceptions import RequestEntityTooLarge from docsgpt.auth import handle_auth from docsgpt.core import log_context +from docsgpt.core.paths import env_file from docsgpt.core.logging_config import setup_logging setup_logging() @@ -53,7 +54,7 @@ if platform.system() == "Windows": import pathlib pathlib.PosixPath = pathlib.WindowsPath -dotenv.load_dotenv() +dotenv.load_dotenv(env_file()) # Self-bootstrap the user-data Postgres DB. Runs before any blueprint or # repository touches the engine, so the first request can't race the diff --git a/docsgpt/cli.py b/docsgpt/cli.py new file mode 100644 index 00000000..4c227f83 --- /dev/null +++ b/docsgpt/cli.py @@ -0,0 +1,145 @@ +"""The ``docsgpt`` command: run the API, the worker and the maintenance scripts. + +Every subcommand imports what it needs when it runs, so ``docsgpt --help`` +stays instant and does not touch the database. +""" + +from __future__ import annotations + +import argparse +import logging +import os +import sys +from typing import Optional, Sequence + +from docsgpt.version import __version__ + +DEFAULT_HOST = "0.0.0.0" +DEFAULT_PORT = 7091 +DEFAULT_QUEUES = "docsgpt,parsing,embeddings" + + +def _api(args: argparse.Namespace) -> int: + """Serve the ASGI app: gunicorn with the bounded-drain uvicorn worker, or uvicorn when reloading.""" + if args.reload or sys.platform == "win32": + import uvicorn + + uvicorn.run("docsgpt.asgi:asgi_app", host=args.host, port=args.port, reload=args.reload) + return 0 + + from gunicorn.app.wsgiapp import run as gunicorn_run + + argv = [ + "gunicorn", + "-w", str(args.workers), + "-k", "docsgpt.gunicorn_worker.BoundedDrainUvicornWorker", + "--bind", f"{args.host}:{args.port}", + "--timeout", "180", + "--graceful-timeout", "120", + "--keep-alive", "5", + "--max-requests", "5000", + "--max-requests-jitter", "500", + "--config", "python:docsgpt.gunicorn_conf", + ] + if os.path.isdir("/dev/shm"): + argv += ["--worker-tmp-dir", "/dev/shm"] + argv.append("docsgpt.asgi:asgi_app") + sys.argv = argv + gunicorn_run() + return 0 + + +def _worker(args: argparse.Namespace) -> int: + """Run the Celery worker (with the beat scheduler unless ``--no-beat``).""" + from docsgpt.app import celery + + argv = ["worker", "-l", args.loglevel, "-Q", args.queues] + if args.concurrency: + argv += ["--concurrency", str(args.concurrency)] + if args.beat: + argv.append("-B") + pool = args.pool or ("solo" if sys.platform == "darwin" else None) + if pool: + argv += ["--pool", pool] + celery.worker_main(argv) + return 0 + + +def _migrate(args: argparse.Namespace) -> int: + """Create the Postgres database if asked and migrate it to head.""" + from docsgpt.core.settings import settings + from docsgpt.storage.db.bootstrap import ensure_database_ready + + logging.basicConfig(level=logging.INFO, format="%(levelname)s %(name)s: %(message)s") + if not settings.POSTGRES_URI: + print("POSTGRES_URI is not set; nothing to migrate.", file=sys.stderr) + return 2 + ensure_database_ready( + settings.POSTGRES_URI, + create_db=args.create_db, + migrate=True, + logger=logging.getLogger("docsgpt.migrate"), + ) + return 0 + + +# Maintenance scripts keep their own argument parsers; the command hands +# everything after the script name to them untouched (argparse would try to +# interpret the options itself). +SCRIPTS = { + "prefetch-models": ("prefetch_models", "download the embedding, tokenizer and parser models"), + "verify-offline": ("verify_offline", "check that the install runs with networking off"), + "reembed": ("reembed", "re-embed every index with the configured embedding model"), +} + + +def _run_script(module: str, argv: list[str]) -> int: + import importlib + + return int(importlib.import_module(f"docsgpt.scripts.{module}").main(argv) or 0) + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(prog="docsgpt", description="DocsGPT: private AI for agents, assistants and search.") + parser.add_argument("--version", action="version", version=f"docsgpt {__version__}") + commands = parser.add_subparsers(dest="command", metavar="") + + api = commands.add_parser("api", help="serve the HTTP API") + api.add_argument("--host", default=DEFAULT_HOST) + api.add_argument("--port", type=int, default=DEFAULT_PORT) + api.add_argument("--workers", type=int, default=1, help="gunicorn worker processes (default: 1)") + api.add_argument("--reload", action="store_true", help="development mode: uvicorn with auto-reload") + api.set_defaults(func=_api) + + worker = commands.add_parser("worker", help="run the Celery worker (and the scheduler)") + worker.add_argument("-Q", "--queues", default=DEFAULT_QUEUES, help=f"queues to consume (default: {DEFAULT_QUEUES})") + worker.add_argument("--concurrency", type=int, help="worker processes (default: one per CPU)") + worker.add_argument("--pool", help="celery pool (default: prefork; solo on macOS)") + worker.add_argument("-l", "--loglevel", default="INFO") + worker.add_argument("--no-beat", dest="beat", action="store_false", help="do not embed the beat scheduler") + worker.set_defaults(func=_worker) + + migrate = commands.add_parser("migrate", help="create the database if needed and run the migrations") + migrate.add_argument("--no-create", dest="create_db", action="store_false", help="fail instead of creating a missing database") + migrate.set_defaults(func=_migrate) + + for name, (module, help_text) in SCRIPTS.items(): + commands.add_parser(name, help=f"{help_text} (docsgpt.scripts.{module})", add_help=False) + + return parser + + +def main(argv: Optional[Sequence[str]] = None) -> int: + argv = list(sys.argv[1:] if argv is None else argv) + if argv and argv[0] in SCRIPTS: + return _run_script(SCRIPTS[argv[0]][0], argv[1:]) + parser = build_parser() + args = parser.parse_args(argv) + if not args.command: + parser.print_help() + return 2 + return args.func(args) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/docsgpt/core/paths.py b/docsgpt/core/paths.py new file mode 100644 index 00000000..48296d21 --- /dev/null +++ b/docsgpt/core/paths.py @@ -0,0 +1,47 @@ +"""Where DocsGPT keeps its files. + +Three locations matter. The package directory holds code and the data that +ships with it (prompts, model catalogs, migrations). A checkout, when the +package is imported from one, is the directory holding ``pyproject.toml``. +The data home is where runtime data lives: ``.env``, ``inputs``, ``indexes``. + +The home is ``DOCSGPT_HOME`` when set, else the checkout, else the current +directory. That keeps a source checkout and the Docker image (which runs from +``/app`` with the package beside it) behaving as before, and gives a +``pip install docsgpt`` user a home that is not ``site-packages``. +""" + +from __future__ import annotations + +import os +from pathlib import Path + +HOME_ENV = "DOCSGPT_HOME" +ENV_FILE_ENV = "DOCSGPT_ENV_FILE" + + +def package_dir() -> Path: + """The installed ``docsgpt`` package directory.""" + return Path(__file__).resolve().parent.parent + + +def checkout_root() -> Path | None: + """The source checkout the package is imported from, or None when installed.""" + root = package_dir().parent + return root if (root / "pyproject.toml").is_file() else None + + +def home_dir() -> Path: + """Directory for runtime data: ``DOCSGPT_HOME``, else the checkout, else cwd.""" + configured = os.environ.get(HOME_ENV) + if configured: + return Path(configured).expanduser().resolve() + return checkout_root() or Path.cwd() + + +def env_file() -> Path: + """The ``.env`` file settings load: ``DOCSGPT_ENV_FILE``, else ``/.env``.""" + configured = os.environ.get(ENV_FILE_ENV) + if configured: + return Path(configured).expanduser() + return home_dir() / ".env" diff --git a/docsgpt/core/settings.py b/docsgpt/core/settings.py index f34841ff..46f3f34b 100644 --- a/docsgpt/core/settings.py +++ b/docsgpt/core/settings.py @@ -1,17 +1,17 @@ import os -from pathlib import Path from typing import Optional from pydantic import AliasChoices, Field, field_validator from pydantic_settings import BaseSettings, SettingsConfigDict -current_dir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) - - -from docsgpt.core.db_uri import ( # noqa: E402 +from docsgpt.core.db_uri import ( normalize_pgvector_connection_string, normalize_postgres_uri, ) +from docsgpt.core.paths import env_file, home_dir + +# Runtime data home (DOCSGPT_HOME, the checkout, or cwd); see docsgpt.core.paths. +current_dir = str(home_dir()) class Settings(BaseSettings): @@ -581,6 +581,4 @@ class Settings(BaseSettings): return stripped -# Project root is one level above application/ -path = Path(__file__).parent.parent.parent.absolute() -settings = Settings(_env_file=path.joinpath(".env"), _env_file_encoding="utf-8") +settings = Settings(_env_file=env_file(), _env_file_encoding="utf-8") diff --git a/docsgpt/storage/local.py b/docsgpt/storage/local.py index ab23e1fb..ba2099c7 100644 --- a/docsgpt/storage/local.py +++ b/docsgpt/storage/local.py @@ -4,6 +4,7 @@ import shutil import tempfile from typing import BinaryIO, List, Callable +from docsgpt.core.paths import home_dir from docsgpt.storage.base import BaseStorage @@ -15,11 +16,10 @@ class LocalStorage(BaseStorage): Initialize local storage. Args: - base_dir: Base directory for all operations. If None, uses current directory. + base_dir: Base directory for all operations. Defaults to the data home + (see ``docsgpt.core.paths.home_dir``). """ - self.base_dir = base_dir or os.path.dirname( - os.path.dirname(os.path.dirname(os.path.abspath(__file__))) - ) + self.base_dir = base_dir or str(home_dir()) def _get_full_path(self, path: str) -> str: """Get absolute path by combining base_dir and path. diff --git a/docsgpt/worker.py b/docsgpt/worker.py index c1c6cb78..b300f1a3 100755 --- a/docsgpt/worker.py +++ b/docsgpt/worker.py @@ -380,10 +380,6 @@ def generate_random_string(length): return "".join([string.ascii_letters[i % 52] for i in range(length)]) -current_dir = os.path.dirname( - os.path.dirname(os.path.dirname(os.path.abspath(__file__))) -) - # Zip extraction security limits. Kept as module constants for backward # compatibility with worker callers/tests; values come from operator settings. MAX_UNCOMPRESSED_SIZE = settings.UPLOAD_MAX_ARCHIVE_BYTES diff --git a/pyproject.toml b/pyproject.toml index d43c281b..3577d6c0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,58 +1,80 @@ +[build-system] +requires = ["hatchling>=1.27,<2"] +build-backend = "hatchling.build" + [project] name = "docsgpt" -# Bump together with docsgpt/version.py and frontend/package.json. -version = "0.19.0" +# The version is read from docsgpt/version.py (the release workflow reads the +# same file); bump it there. +dynamic = ["version"] description = "DocsGPT backend: chat with your documents, agents, and tools." readme = "README.md" requires-python = ">=3.12" -license = { file = "LICENSE" } +license = "MIT" +license-files = ["LICENSE"] +authors = [{ name = "Arc53" }] +keywords = ["rag", "llm", "agents", "documents", "chat", "search"] +classifiers = [ + "Development Status :: 5 - Production/Stable", + "Environment :: Web Environment", + "Framework :: Flask", + "Intended Audience :: Developers", + "Operating System :: OS Independent", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Topic :: Scientific/Engineering :: Artificial Intelligence", +] -# Direct dependencies only. Transitive pins live in uv.lock; the pip-facing -# files under docsgpt/ (requirements*.txt) are exported from that lock by -# scripts/export_requirements.sh and must not be edited by hand. +# Direct dependencies only, as compatible ranges: the floor is the version +# uv.lock pins, the ceiling the next major (next minor below 1.0), so the +# PyPI package installs next to other packages. The pip-facing files under +# docsgpt/ (requirements*.txt) are exported from the lock by +# scripts/export_requirements.sh and must not be edited by hand; Docker, CI +# and the checkout install those exact versions. dependencies = [ - "a2wsgi==1.10.10", + "a2wsgi>=1.10.10,<2", "alembic>=1.13,<2", - "anthropic==0.121.0", - "beautifulsoup4==4.15.0", - "boto3==1.43.67", - "cel-python==0.5.0", - "celery==5.6.3", - "celery-redbeat==2.4.2", - "croniter==6.2.4", - "cryptography==50.0.0", - "dataclasses-json==0.6.7", - "daytona==0.205.1", - "ddgs>=8.0.0", - "defusedxml==0.7.1", - "docx2txt==0.9", - "elevenlabs==2.62.0", - "faiss-cpu==1.15.0", - "fast-ebook", + "anthropic>=0.121.0,<0.122", + "beautifulsoup4>=4.15.0,<5", + "boto3>=1.43.67,<2", + "cel-python>=0.5.0,<0.6", + "celery>=5.6.3,<6", + "celery-redbeat>=2.4.2,<3", + "croniter>=6.2.4,<7", + "cryptography>=50.0.0,<51", + "dataclasses-json>=0.6.7,<0.7", + "daytona>=0.205.1,<0.206", + "ddgs>=8.0.0,<10", + "defusedxml>=0.7.1,<0.8", + "docx2txt>=0.9,<0.10", + "elevenlabs>=2.62.0,<3", + "faiss-cpu>=1.15.0,<2", + "fast-ebook>=0.2.0,<0.3", # Default document converter (DOC_PARSER_ENGINE=anydoc): a Rust extension # with no model downloads. The docling engine is the `docling` extra. - "firecrawl-anydoc==0.2.3", - "fastembed==0.8.0", - "fastmcp==3.4.6", - "Flask==3.1.3", - "flask-restx==1.3.2", - "google-api-python-client==2.198.0", - "google-auth-oauthlib==1.4.0", - "google-genai==2.17.0", - "gTTS==2.5.4", - "gunicorn==26.0.0", - "jinja2==3.1.6", - "kombu==5.6.2", - "markdownify==1.2.3", - "msal==1.37.0", - "networkx==3.6.1", - "numpy==2.5.1", + "firecrawl-anydoc>=0.2.3,<0.3", + "fastembed>=0.8.0,<0.9", + "fastmcp>=3.4.6,<4", + "Flask>=3.1.3,<4", + "flask-restx>=1.3.2,<2", + "google-api-python-client>=2.198.0,<3", + "google-auth-oauthlib>=1.4.0,<2", + "google-genai>=2.17.0,<3", + "gTTS>=2.5.4,<3", + "gunicorn>=26.0.0,<27", + "jinja2>=3.1.6,<4", + "kombu>=5.6.2,<6", + "markdownify>=1.2.3,<2", + "msal>=1.37.0,<2", + "networkx>=3.6.1,<4", + "numpy>=2.5.1,<3", # fastembed's runtime: local embeddings execute on it. - "onnxruntime==1.28.0", - "openai==2.53.0", - "openapi3-parser==1.1.22", + "onnxruntime>=1.28.0,<2", + "openai>=2.53.0,<3", + "openapi3-parser>=1.1.22,<2", # pandas reads .xlsx through openpyxl but does not depend on it. - "openpyxl==3.1.5", + "openpyxl>=3.1.5,<4", "opentelemetry-distro>=0.50b0,<1", "opentelemetry-exporter-otlp>=1.29.0,<2", "opentelemetry-instrumentation-celery>=0.50b0,<1", @@ -63,59 +85,71 @@ dependencies = [ "opentelemetry-instrumentation-requests>=0.50b0,<1", "opentelemetry-instrumentation-sqlalchemy>=0.50b0,<1", "opentelemetry-instrumentation-starlette>=0.50b0,<1", - "pandas==3.0.5", - "pdf2image>=1.17.0", + "pandas>=3.0.5,<4", + "pdf2image>=1.17.0,<2", "pgvector>=0.5,<1", - "pillow==12.3.0", - "praw==8.0.2", + "pillow>=12.3.0,<13", + "praw>=8.0.2,<9", "psycopg[binary,pool]>=3.1,<4", - "pydantic", - "pydantic-settings", - "pypdf==6.15.0", - "pypdfium2==5.12.1", - "python-dateutil==2.9.0.post0", - "python-dotenv", - "python-jose==3.5.0", - "python-pptx==1.0.2", - "PyYAML", - "qdrant-client==1.19.0", - "redis==7.4.0", - "requests==2.34.2", - "retry==0.9.2", + "pydantic>=2.13.5,<3", + "pydantic-settings>=2.15.0,<3", + "pypdf>=6.15.0,<7", + "pypdfium2>=5.12.1,<6", + "python-dateutil>=2.9.0,<3", + "python-dotenv>=1.2.3,<2", + "python-jose>=3.5.0,<4", + "python-pptx>=1.0.2,<2", + "PyYAML>=6.0.3,<7", + "qdrant-client>=1.19.0,<2", + "redis>=7.4.0,<8", + "requests>=2.34.2,<3", + "retry>=0.9.2,<0.10", "sqlalchemy>=2.0,<3", "starlette>=1.0,<2", - "tiktoken==0.13.0", - "tldextract==5.3.2", - "tokenizers==0.22.2", - "tqdm==4.67.3", + "tiktoken>=0.13.0,<0.14", + "tldextract>=5.3.2,<6", + "tokenizers>=0.22.2,<0.23", + "tqdm>=4.67.3,<5", "uvicorn[standard]>=0.30,<1", "uvicorn-worker>=0.4,<1", - "websocket-client==1.9.0", - "werkzeug>=3.1.0", + "websocket-client>=1.9.0,<2", + "werkzeug>=3.1.0,<4", ] [project.optional-dependencies] # Docling parser engine: DOC_PARSER_ENGINE=docling, the docling OCR backend # (layout-model hybrid OCR, ocrmac/rapidocr engines), .adoc/.vtt/.xml # attachment parsing, and read_document's `structured` output. Pulls torch and -# transformers; on Linux torch resolves from the CPU-only PyTorch index (see -# [tool.uv.sources]) so the extra does not drag the CUDA stack in. +# transformers; with uv on Linux torch resolves from the CPU-only PyTorch index +# (see [tool.uv.sources]) so the extra does not drag the CUDA stack in. pip +# users on Linux get the same by adding +# --extra-index-url https://download.pytorch.org/whl/cpu. docling = [ - "docling==2.119.0", - "rapidocr==3.9.2", - # docling's model stack. Declared here (not left transitive) so the pins + "docling>=2.119.0,<3", + "rapidocr>=3.9.2,<4", + # docling's model stack. Declared here (not left transitive) so the floors # hold and the CPU index source below applies. transformers is capped by # docling-core at <5.9: 5.9+ breaks the PDF layout model on Apple Silicon. - "torch==2.11.0", - "torchvision==0.26.0", - "transformers==5.8.1", + "torch>=2.11.0,<3", + "torchvision>=0.26.0,<0.27", + "transformers>=5.8.1,<5.9", ] # VECTOR_STORE=milvus. milvus-lite (the embedded server) pulls pyarrow. milvus = [ - "pymilvus==3.0.1", - "milvus-lite==3.2.0; sys_platform != 'win32'", + "pymilvus>=3.0.1,<4", + "milvus-lite>=3.2.0,<4; sys_platform != 'win32'", ] +[project.scripts] +docsgpt = "docsgpt.cli:main" + +[project.urls] +Homepage = "https://www.docsgpt.cloud/" +Documentation = "https://docs.docsgpt.cloud/" +Repository = "https://github.com/arc53/DocsGPT" +Issues = "https://github.com/arc53/DocsGPT/issues" +Changelog = "https://github.com/arc53/DocsGPT/releases" + [dependency-groups] # Mirrors tests/requirements.txt for `uv sync`; pip users install that file. dev = [ @@ -131,10 +165,36 @@ dev = [ "ruff", ] -[tool.uv] -# The repo is run in place (`application.*` imported from the checkout), not -# installed as a distribution. -package = false +[tool.hatch.version] +path = "docsgpt/version.py" + +# The wheel is the docsgpt package with the data it reads at runtime: prompts, +# model catalogs, the seed config, alembic.ini and the migrations. Build inputs +# (Dockerfile, exported requirements), the sample index and local runtime data +# stay out. The one-release `application` import alias is checkout-only. +[tool.hatch.build.targets.wheel] +packages = ["docsgpt"] +exclude = [ + "docsgpt/Dockerfile", + "docsgpt/requirements*.txt", + "docsgpt/index.faiss", + "docsgpt/index.pkl", + "docsgpt/indexes/", + "docsgpt/inputs/", + "docsgpt/vectors/", +] + +[tool.hatch.build.targets.sdist] +include = ["/docsgpt"] +exclude = [ + "docsgpt/Dockerfile", + "docsgpt/requirements*.txt", + "docsgpt/index.faiss", + "docsgpt/index.pkl", + "docsgpt/indexes/", + "docsgpt/inputs/", + "docsgpt/vectors/", +] [[tool.uv.index]] name = "pytorch-cpu" diff --git a/tests/core/test_paths.py b/tests/core/test_paths.py new file mode 100644 index 00000000..54c74d19 --- /dev/null +++ b/tests/core/test_paths.py @@ -0,0 +1,43 @@ +"""Data home and .env discovery (docsgpt.core.paths).""" + +from pathlib import Path + +from docsgpt.core import paths + +REPO_ROOT = Path(__file__).resolve().parents[2] + + +class TestHomeDir: + def test_the_checkout_wins_over_cwd(self, monkeypatch, tmp_path): + monkeypatch.delenv(paths.HOME_ENV, raising=False) + monkeypatch.chdir(tmp_path) + assert paths.checkout_root() == REPO_ROOT + assert paths.home_dir() == REPO_ROOT + + def test_the_env_var_wins(self, monkeypatch, tmp_path): + monkeypatch.setenv(paths.HOME_ENV, str(tmp_path)) + assert paths.home_dir() == tmp_path.resolve() + + def test_an_installed_package_falls_back_to_cwd(self, monkeypatch, tmp_path): + monkeypatch.delenv(paths.HOME_ENV, raising=False) + monkeypatch.setattr(paths, "checkout_root", lambda: None) + monkeypatch.chdir(tmp_path) + assert paths.home_dir() == Path.cwd() + + +class TestEnvFile: + def test_default_is_dot_env_in_the_home(self, monkeypatch, tmp_path): + monkeypatch.delenv(paths.ENV_FILE_ENV, raising=False) + monkeypatch.setenv(paths.HOME_ENV, str(tmp_path)) + assert paths.env_file() == tmp_path.resolve() / ".env" + + def test_the_env_var_wins(self, monkeypatch, tmp_path): + monkeypatch.setenv(paths.ENV_FILE_ENV, str(tmp_path / "custom.env")) + assert paths.env_file() == tmp_path / "custom.env" + + +class TestPackageDir: + def test_holds_the_shipped_data(self): + assert paths.package_dir().name == "docsgpt" + assert (paths.package_dir() / "alembic.ini").is_file() + assert (paths.package_dir() / "prompts" / "chat_reduce_prompt.txt").is_file() diff --git a/tests/test_cli.py b/tests/test_cli.py new file mode 100644 index 00000000..00d2c8bc --- /dev/null +++ b/tests/test_cli.py @@ -0,0 +1,111 @@ +"""The ``docsgpt`` command dispatches to the API server, the worker and the scripts.""" + +import subprocess +import sys +import types +from pathlib import Path +from unittest.mock import MagicMock + +import pytest + +from docsgpt import cli +from docsgpt.version import __version__ + + +class TestTopLevel: + def test_version(self, capsys): + with pytest.raises(SystemExit) as exc: + cli.main(["--version"]) + assert exc.value.code == 0 + assert capsys.readouterr().out.strip() == f"docsgpt {__version__}" + + def test_no_command_prints_help(self, capsys): + assert cli.main([]) == 2 + assert "worker" in capsys.readouterr().out + + def test_importing_the_cli_does_not_boot_the_app(self): + """``docsgpt --help`` must not import the Flask app, Celery or settings.""" + code = ( + "import sys, docsgpt.cli; " + "loaded = {m for m in sys.modules if m in ('docsgpt.app', 'docsgpt.core.settings', 'celery', 'flask')}; " + "assert not loaded, loaded" + ) + subprocess.run([sys.executable, "-c", code], cwd=Path(__file__).resolve().parents[1], check=True) + + +class TestApi: + def test_gunicorn_argv(self, monkeypatch): + run = MagicMock() + monkeypatch.setitem(sys.modules, "gunicorn.app.wsgiapp", types.SimpleNamespace(run=run)) + monkeypatch.setattr(sys, "platform", "linux") + monkeypatch.setattr(sys, "argv", ["docsgpt"]) + assert cli.main(["api", "--port", "8000", "--workers", "2"]) == 0 + run.assert_called_once_with() + argv = sys.argv + assert argv[0] == "gunicorn" and argv[-1] == "docsgpt.asgi:asgi_app" + assert argv[argv.index("--bind") + 1] == "0.0.0.0:8000" + assert argv[argv.index("-w") + 1] == "2" + assert argv[argv.index("-k") + 1] == "docsgpt.gunicorn_worker.BoundedDrainUvicornWorker" + assert argv[argv.index("--config") + 1] == "python:docsgpt.gunicorn_conf" + + def test_reload_uses_uvicorn(self, monkeypatch): + uvicorn = types.SimpleNamespace(run=MagicMock()) + monkeypatch.setitem(sys.modules, "uvicorn", uvicorn) + assert cli.main(["api", "--reload", "--host", "127.0.0.1"]) == 0 + uvicorn.run.assert_called_once_with("docsgpt.asgi:asgi_app", host="127.0.0.1", port=7091, reload=True) + + +class TestWorker: + @staticmethod + def _celery(monkeypatch): + celery = MagicMock() + monkeypatch.setitem(sys.modules, "docsgpt.app", types.SimpleNamespace(celery=celery)) + return celery + + def test_defaults(self, monkeypatch): + celery = self._celery(monkeypatch) + monkeypatch.setattr(sys, "platform", "linux") + assert cli.main(["worker"]) == 0 + argv = celery.worker_main.call_args.args[0] + assert argv[:3] == ["worker", "-l", "INFO"] + assert argv[argv.index("-Q") + 1] == cli.DEFAULT_QUEUES + assert "-B" in argv + assert "--pool" not in argv + + def test_options_and_the_macos_solo_pool(self, monkeypatch): + celery = self._celery(monkeypatch) + monkeypatch.setattr(sys, "platform", "darwin") + assert cli.main(["worker", "--no-beat", "-Q", "embeddings", "--concurrency", "2"]) == 0 + argv = celery.worker_main.call_args.args[0] + assert argv[argv.index("--pool") + 1] == "solo" + assert "-B" not in argv + assert argv[argv.index("-Q") + 1] == "embeddings" + assert argv[argv.index("--concurrency") + 1] == "2" + + +class TestMigrate: + def test_runs_the_bootstrap(self, monkeypatch): + ensure = MagicMock() + monkeypatch.setattr("docsgpt.storage.db.bootstrap.ensure_database_ready", ensure) + monkeypatch.setattr("docsgpt.core.settings.settings.POSTGRES_URI", "postgresql://docsgpt@localhost/docsgpt") + assert cli.main(["migrate", "--no-create"]) == 0 + assert ensure.call_args.args[0] == "postgresql://docsgpt@localhost/docsgpt" + assert ensure.call_args.kwargs["create_db"] is False + assert ensure.call_args.kwargs["migrate"] is True + + def test_without_a_database_uri(self, monkeypatch, capsys): + monkeypatch.setattr("docsgpt.core.settings.settings.POSTGRES_URI", None) + assert cli.main(["migrate"]) == 2 + assert "POSTGRES_URI" in capsys.readouterr().err + + +class TestScripts: + def test_arguments_pass_through_untouched(self, monkeypatch): + main = MagicMock(return_value=0) + monkeypatch.setattr("docsgpt.scripts.prefetch_models.main", main) + assert cli.main(["prefetch-models", "--embeddings", "x", "--help"]) == 0 + main.assert_called_once_with(["--embeddings", "x", "--help"]) + + def test_the_script_exit_code_is_returned(self, monkeypatch): + monkeypatch.setattr("docsgpt.scripts.verify_offline.main", MagicMock(return_value=3)) + assert cli.main(["verify-offline"]) == 3 diff --git a/uv.lock b/uv.lock index f120d37d..fe9b0717 100644 --- a/uv.lock +++ b/uv.lock @@ -1385,8 +1385,7 @@ standard = [ [[package]] name = "docsgpt" -version = "0.19.0" -source = { virtual = "." } +source = { editable = "." } dependencies = [ { name = "a2wsgi" }, { name = "alembic" }, @@ -1498,46 +1497,46 @@ dev = [ [package.metadata] requires-dist = [ - { name = "a2wsgi", specifier = "==1.10.10" }, + { name = "a2wsgi", specifier = ">=1.10.10,<2" }, { name = "alembic", specifier = ">=1.13,<2" }, - { name = "anthropic", specifier = "==0.121.0" }, - { name = "beautifulsoup4", specifier = "==4.15.0" }, - { name = "boto3", specifier = "==1.43.67" }, - { name = "cel-python", specifier = "==0.5.0" }, - { name = "celery", specifier = "==5.6.3" }, - { name = "celery-redbeat", specifier = "==2.4.2" }, - { name = "croniter", specifier = "==6.2.4" }, - { name = "cryptography", specifier = "==50.0.0" }, - { name = "dataclasses-json", specifier = "==0.6.7" }, - { name = "daytona", specifier = "==0.205.1" }, - { name = "ddgs", specifier = ">=8.0.0" }, - { name = "defusedxml", specifier = "==0.7.1" }, - { name = "docling", marker = "extra == 'docling'", specifier = "==2.119.0" }, - { name = "docx2txt", specifier = "==0.9" }, - { name = "elevenlabs", specifier = "==2.62.0" }, - { name = "faiss-cpu", specifier = "==1.15.0" }, - { name = "fast-ebook" }, - { name = "fastembed", specifier = "==0.8.0" }, - { name = "fastmcp", specifier = "==3.4.6" }, - { name = "firecrawl-anydoc", specifier = "==0.2.3" }, - { name = "flask", specifier = "==3.1.3" }, - { name = "flask-restx", specifier = "==1.3.2" }, - { name = "google-api-python-client", specifier = "==2.198.0" }, - { name = "google-auth-oauthlib", specifier = "==1.4.0" }, - { name = "google-genai", specifier = "==2.17.0" }, - { name = "gtts", specifier = "==2.5.4" }, - { name = "gunicorn", specifier = "==26.0.0" }, - { name = "jinja2", specifier = "==3.1.6" }, - { name = "kombu", specifier = "==5.6.2" }, - { name = "markdownify", specifier = "==1.2.3" }, - { name = "milvus-lite", marker = "sys_platform != 'win32' and extra == 'milvus'", specifier = "==3.2.0" }, - { name = "msal", specifier = "==1.37.0" }, - { name = "networkx", specifier = "==3.6.1" }, - { name = "numpy", specifier = "==2.5.1" }, - { name = "onnxruntime", specifier = "==1.28.0" }, - { name = "openai", specifier = "==2.53.0" }, - { name = "openapi3-parser", specifier = "==1.1.22" }, - { name = "openpyxl", specifier = "==3.1.5" }, + { name = "anthropic", specifier = ">=0.121.0,<0.122" }, + { name = "beautifulsoup4", specifier = ">=4.15.0,<5" }, + { name = "boto3", specifier = ">=1.43.67,<2" }, + { name = "cel-python", specifier = ">=0.5.0,<0.6" }, + { name = "celery", specifier = ">=5.6.3,<6" }, + { name = "celery-redbeat", specifier = ">=2.4.2,<3" }, + { name = "croniter", specifier = ">=6.2.4,<7" }, + { name = "cryptography", specifier = ">=50.0.0,<51" }, + { name = "dataclasses-json", specifier = ">=0.6.7,<0.7" }, + { name = "daytona", specifier = ">=0.205.1,<0.206" }, + { name = "ddgs", specifier = ">=8.0.0,<10" }, + { name = "defusedxml", specifier = ">=0.7.1,<0.8" }, + { name = "docling", marker = "extra == 'docling'", specifier = ">=2.119.0,<3" }, + { name = "docx2txt", specifier = ">=0.9,<0.10" }, + { name = "elevenlabs", specifier = ">=2.62.0,<3" }, + { name = "faiss-cpu", specifier = ">=1.15.0,<2" }, + { name = "fast-ebook", specifier = ">=0.2.0,<0.3" }, + { name = "fastembed", specifier = ">=0.8.0,<0.9" }, + { name = "fastmcp", specifier = ">=3.4.6,<4" }, + { name = "firecrawl-anydoc", specifier = ">=0.2.3,<0.3" }, + { name = "flask", specifier = ">=3.1.3,<4" }, + { name = "flask-restx", specifier = ">=1.3.2,<2" }, + { name = "google-api-python-client", specifier = ">=2.198.0,<3" }, + { name = "google-auth-oauthlib", specifier = ">=1.4.0,<2" }, + { name = "google-genai", specifier = ">=2.17.0,<3" }, + { name = "gtts", specifier = ">=2.5.4,<3" }, + { name = "gunicorn", specifier = ">=26.0.0,<27" }, + { name = "jinja2", specifier = ">=3.1.6,<4" }, + { name = "kombu", specifier = ">=5.6.2,<6" }, + { name = "markdownify", specifier = ">=1.2.3,<2" }, + { name = "milvus-lite", marker = "sys_platform != 'win32' and extra == 'milvus'", specifier = ">=3.2.0,<4" }, + { name = "msal", specifier = ">=1.37.0,<2" }, + { name = "networkx", specifier = ">=3.6.1,<4" }, + { name = "numpy", specifier = ">=2.5.1,<3" }, + { name = "onnxruntime", specifier = ">=1.28.0,<2" }, + { name = "openai", specifier = ">=2.53.0,<3" }, + { name = "openapi3-parser", specifier = ">=1.1.22,<2" }, + { name = "openpyxl", specifier = ">=3.1.5,<4" }, { name = "opentelemetry-distro", specifier = ">=0.50b0,<1" }, { name = "opentelemetry-exporter-otlp", specifier = ">=1.29.0,<2" }, { name = "opentelemetry-instrumentation-celery", specifier = ">=0.50b0,<1" }, @@ -1548,42 +1547,42 @@ requires-dist = [ { name = "opentelemetry-instrumentation-requests", specifier = ">=0.50b0,<1" }, { name = "opentelemetry-instrumentation-sqlalchemy", specifier = ">=0.50b0,<1" }, { name = "opentelemetry-instrumentation-starlette", specifier = ">=0.50b0,<1" }, - { name = "pandas", specifier = "==3.0.5" }, - { name = "pdf2image", specifier = ">=1.17.0" }, + { name = "pandas", specifier = ">=3.0.5,<4" }, + { name = "pdf2image", specifier = ">=1.17.0,<2" }, { name = "pgvector", specifier = ">=0.5,<1" }, - { name = "pillow", specifier = "==12.3.0" }, - { name = "praw", specifier = "==8.0.2" }, + { name = "pillow", specifier = ">=12.3.0,<13" }, + { name = "praw", specifier = ">=8.0.2,<9" }, { name = "psycopg", extras = ["binary", "pool"], specifier = ">=3.1,<4" }, - { name = "pydantic" }, - { name = "pydantic-settings" }, - { name = "pymilvus", marker = "extra == 'milvus'", specifier = "==3.0.1" }, - { name = "pypdf", specifier = "==6.15.0" }, - { name = "pypdfium2", specifier = "==5.12.1" }, - { name = "python-dateutil", specifier = "==2.9.0.post0" }, - { name = "python-dotenv" }, - { name = "python-jose", specifier = "==3.5.0" }, - { name = "python-pptx", specifier = "==1.0.2" }, - { name = "pyyaml" }, - { name = "qdrant-client", specifier = "==1.19.0" }, - { name = "rapidocr", marker = "extra == 'docling'", specifier = "==3.9.2" }, - { name = "redis", specifier = "==7.4.0" }, - { name = "requests", specifier = "==2.34.2" }, - { name = "retry", specifier = "==0.9.2" }, + { name = "pydantic", specifier = ">=2.13.5,<3" }, + { name = "pydantic-settings", specifier = ">=2.15.0,<3" }, + { name = "pymilvus", marker = "extra == 'milvus'", specifier = ">=3.0.1,<4" }, + { name = "pypdf", specifier = ">=6.15.0,<7" }, + { name = "pypdfium2", specifier = ">=5.12.1,<6" }, + { name = "python-dateutil", specifier = ">=2.9.0,<3" }, + { name = "python-dotenv", specifier = ">=1.2.3,<2" }, + { name = "python-jose", specifier = ">=3.5.0,<4" }, + { name = "python-pptx", specifier = ">=1.0.2,<2" }, + { name = "pyyaml", specifier = ">=6.0.3,<7" }, + { name = "qdrant-client", specifier = ">=1.19.0,<2" }, + { name = "rapidocr", marker = "extra == 'docling'", specifier = ">=3.9.2,<4" }, + { name = "redis", specifier = ">=7.4.0,<8" }, + { name = "requests", specifier = ">=2.34.2,<3" }, + { name = "retry", specifier = ">=0.9.2,<0.10" }, { name = "sqlalchemy", specifier = ">=2.0,<3" }, { name = "starlette", specifier = ">=1.0,<2" }, - { name = "tiktoken", specifier = "==0.13.0" }, - { name = "tldextract", specifier = "==5.3.2" }, - { name = "tokenizers", specifier = "==0.22.2" }, - { name = "torch", marker = "sys_platform == 'linux' and extra == 'docling'", specifier = "==2.11.0", index = "https://download.pytorch.org/whl/cpu" }, - { name = "torch", marker = "sys_platform != 'linux' and extra == 'docling'", specifier = "==2.11.0" }, - { name = "torchvision", marker = "sys_platform == 'linux' and extra == 'docling'", specifier = "==0.26.0", index = "https://download.pytorch.org/whl/cpu" }, - { name = "torchvision", marker = "sys_platform != 'linux' and extra == 'docling'", specifier = "==0.26.0" }, - { name = "tqdm", specifier = "==4.67.3" }, - { name = "transformers", marker = "extra == 'docling'", specifier = "==5.8.1" }, + { name = "tiktoken", specifier = ">=0.13.0,<0.14" }, + { name = "tldextract", specifier = ">=5.3.2,<6" }, + { name = "tokenizers", specifier = ">=0.22.2,<0.23" }, + { name = "torch", marker = "sys_platform == 'linux' and extra == 'docling'", specifier = ">=2.11.0,<3", index = "https://download.pytorch.org/whl/cpu" }, + { name = "torch", marker = "sys_platform != 'linux' and extra == 'docling'", specifier = ">=2.11.0,<3" }, + { name = "torchvision", marker = "sys_platform == 'linux' and extra == 'docling'", specifier = ">=0.26.0,<0.27", index = "https://download.pytorch.org/whl/cpu" }, + { name = "torchvision", marker = "sys_platform != 'linux' and extra == 'docling'", specifier = ">=0.26.0,<0.27" }, + { name = "tqdm", specifier = ">=4.67.3,<5" }, + { name = "transformers", marker = "extra == 'docling'", specifier = ">=5.8.1,<5.9" }, { name = "uvicorn", extras = ["standard"], specifier = ">=0.30,<1" }, { name = "uvicorn-worker", specifier = ">=0.4,<1" }, - { name = "websocket-client", specifier = "==1.9.0" }, - { name = "werkzeug", specifier = ">=3.1.0" }, + { name = "websocket-client", specifier = ">=1.9.0,<2" }, + { name = "werkzeug", specifier = ">=3.1.0,<4" }, ] provides-extras = ["docling", "milvus"] From ae9348bb2a2cad99b8e89ea62db056309dafb208 Mon Sep 17 00:00:00 2001 From: Alex Date: Mon, 7 Sep 2026 15:25:30 +0100 Subject: [PATCH 2/4] fix(package): review pass on the PyPI package - docsgpt api binds 127.0.0.1 by default, like gunicorn and uvicorn do; --host 0.0.0.0 exposes it. The docs say so. - The embedded Milvus and LanceDB defaults derive from the data home, so they follow DOCSGPT_HOME like the faiss indexes and uploads do. A checkout run from its root and the Docker image resolve to the same paths as before. - The docs and the pyproject comment describe the CPU torch install as two steps (torch and torchvision from the PyTorch CPU index first, then the docling extra): pip picks the highest version across indexes, so --extra-index-url only yields the CPU build while that index keeps pace with PyPI. - AGENTS.md separates DOCSGPT_HOME (moves the data home) from DOCSGPT_ENV_FILE (selects the .env file); the docs example uses a password placeholder. --- AGENTS.md | 5 +++-- docs/content/Deploying/Pip-Install.mdx | 15 ++++++++++++--- docsgpt/cli.py | 4 ++-- docsgpt/core/settings.py | 5 +++-- pyproject.toml | 5 +++-- tests/core/test_paths.py | 19 +++++++++++++++++++ tests/test_cli.py | 2 +- 7 files changed, 43 insertions(+), 12 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 8b878d56..d096cf22 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -41,8 +41,9 @@ The backend is also an installable package (`pyproject.toml`, hatchling). `uv sync` installs it editable and puts a `docsgpt` command on PATH: `docsgpt api --reload`, `docsgpt worker`, `docsgpt migrate`, `docsgpt prefetch-models`, `docsgpt verify-offline`. Runtime data (`.env`, -`inputs/`, `indexes/`) lives in the checkout by default; `DOCSGPT_HOME` and -`DOCSGPT_ENV_FILE` move it (see `docsgpt/core/paths.py`). +`inputs/`, `indexes/`) lives in the checkout by default; `DOCSGPT_HOME` moves +that data home, and `DOCSGPT_ENV_FILE` selects only the `.env` file (see +`docsgpt/core/paths.py`). Dependencies are declared in `pyproject.toml` and locked in `uv.lock`; the `docsgpt/requirements*.txt` files are exported from the lock. To add or diff --git a/docs/content/Deploying/Pip-Install.mdx b/docs/content/Deploying/Pip-Install.mdx index 41586200..84358c96 100644 --- a/docs/content/Deploying/Pip-Install.mdx +++ b/docs/content/Deploying/Pip-Install.mdx @@ -34,7 +34,14 @@ pip install "docsgpt[docling]" # DOC_PARSER_ENGINE=docling: OCR backend, struc pip install "docsgpt[milvus]" # VECTOR_STORE=milvus ``` -The `docling` extra pulls in PyTorch. On Linux, PyPI's torch wheels bring the CUDA stack with them; on a CPU-only machine add `--extra-index-url https://download.pytorch.org/whl/cpu` to the install command to get the CPU build instead. `uv pip install` and `pipx install` work the same way. +The `docling` extra pulls in PyTorch, and on Linux the PyPI torch wheels bring the CUDA stack with them. On a CPU-only machine install the CPU build first, then the extra; pip keeps the torch it already has: + +```bash +pip install --index-url https://download.pytorch.org/whl/cpu torch torchvision +pip install "docsgpt[docling]" +``` + +`uv pip install` accepts the same two commands. With `pipx`, install `docsgpt[docling]` and then run `pipx inject --index-url https://download.pytorch.org/whl/cpu docsgpt torch torchvision` to swap the CUDA build for the CPU one. ## Configure @@ -49,7 +56,7 @@ LLM_NAME= API_KEY= # User data -POSTGRES_URI=postgresql://docsgpt:docsgpt@localhost:5432/docsgpt +POSTGRES_URI=postgresql://docsgpt:@localhost:5432/docsgpt # Worker-to-API authentication (required for uploads) INTERNAL_KEY= @@ -61,10 +68,12 @@ Redis defaults to `localhost:6379` (`CELERY_BROKER_URL`, `CELERY_RESULT_BACKEND` ```bash docsgpt migrate # create the database if it is missing and apply the migrations -docsgpt api # the API on http://localhost:7091 +docsgpt api # the API on http://127.0.0.1:7091 docsgpt worker # in a second terminal: the Celery worker, with the scheduler ``` +`docsgpt api` listens on localhost only. Pass `--host 0.0.0.0` to accept connections from other machines or containers, and put a reverse proxy with TLS in front of it for anything public. + The API applies pending migrations when it starts (`AUTO_MIGRATE`), so `docsgpt migrate` is the explicit step for deployments that want the schema in place before the first request or that run the API with a restricted database role. The worker is not optional: query embedding runs on it, so search fails without one. `docsgpt worker --help` lists the queue, concurrency and pool options; `--no-beat` starts a worker without the scheduler when another worker already runs it. diff --git a/docsgpt/cli.py b/docsgpt/cli.py index 4c227f83..046ed9f9 100644 --- a/docsgpt/cli.py +++ b/docsgpt/cli.py @@ -14,7 +14,7 @@ from typing import Optional, Sequence from docsgpt.version import __version__ -DEFAULT_HOST = "0.0.0.0" +DEFAULT_HOST = "127.0.0.1" DEFAULT_PORT = 7091 DEFAULT_QUEUES = "docsgpt,parsing,embeddings" @@ -105,7 +105,7 @@ def build_parser() -> argparse.ArgumentParser: commands = parser.add_subparsers(dest="command", metavar="") api = commands.add_parser("api", help="serve the HTTP API") - api.add_argument("--host", default=DEFAULT_HOST) + api.add_argument("--host", default=DEFAULT_HOST, help="interface to listen on (default: localhost; 0.0.0.0 for all)") api.add_argument("--port", type=int, default=DEFAULT_PORT) api.add_argument("--workers", type=int, default=1, help="gunicorn worker processes (default: 1)") api.add_argument("--reload", action="store_true", help="development mode: uvicorn with auto-reload") diff --git a/docsgpt/core/settings.py b/docsgpt/core/settings.py index 46f3f34b..f926485b 100644 --- a/docsgpt/core/settings.py +++ b/docsgpt/core/settings.py @@ -326,11 +326,12 @@ class Settings(BaseSettings): PGVECTOR_IVFFLAT_PROBES: Optional[int] = None # Milvus vectorstore config MILVUS_COLLECTION_NAME: Optional[str] = "docsgpt" - MILVUS_URI: Optional[str] = "./milvus_local.db" # milvus lite version as default + # milvus-lite (embedded) database file, under the data home like the other local stores + MILVUS_URI: Optional[str] = Field(default_factory=lambda: str(home_dir() / "milvus_local.db")) MILVUS_TOKEN: Optional[str] = "" # LanceDB vectorstore config - LANCEDB_PATH: str = "./data/lancedb" # Path where LanceDB stores its local data + LANCEDB_PATH: str = Field(default_factory=lambda: str(home_dir() / "data" / "lancedb")) # LanceDB local data LANCEDB_TABLE_NAME: Optional[str] = "docsgpts" # Name of the table to use for storing vectors FLASK_DEBUG_MODE: bool = False diff --git a/pyproject.toml b/pyproject.toml index 3577d6c0..99fcf8a0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -122,8 +122,9 @@ dependencies = [ # attachment parsing, and read_document's `structured` output. Pulls torch and # transformers; with uv on Linux torch resolves from the CPU-only PyTorch index # (see [tool.uv.sources]) so the extra does not drag the CUDA stack in. pip -# users on Linux get the same by adding -# --extra-index-url https://download.pytorch.org/whl/cpu. +# users on Linux install torch and torchvision from that index first +# (--index-url https://download.pytorch.org/whl/cpu), then the extra; pip keeps +# the torch it already has. docling = [ "docling>=2.119.0,<3", "rapidocr>=3.9.2,<4", diff --git a/tests/core/test_paths.py b/tests/core/test_paths.py index 54c74d19..f6cb7a0b 100644 --- a/tests/core/test_paths.py +++ b/tests/core/test_paths.py @@ -36,6 +36,25 @@ class TestEnvFile: assert paths.env_file() == tmp_path / "custom.env" +class TestSettingsFollowTheHome: + def test_embedded_vector_stores_default_under_the_home(self, monkeypatch, tmp_path): + from docsgpt.core.settings import Settings + + monkeypatch.setenv(paths.HOME_ENV, str(tmp_path)) + monkeypatch.delenv("MILVUS_URI", raising=False) + monkeypatch.delenv("LANCEDB_PATH", raising=False) + fresh = Settings(_env_file=None) + assert fresh.MILVUS_URI == str(tmp_path.resolve() / "milvus_local.db") + assert fresh.LANCEDB_PATH == str(tmp_path.resolve() / "data" / "lancedb") + + def test_env_overrides_still_win(self, monkeypatch, tmp_path): + from docsgpt.core.settings import Settings + + monkeypatch.setenv(paths.HOME_ENV, str(tmp_path)) + monkeypatch.setenv("LANCEDB_PATH", "/srv/lancedb") + assert Settings(_env_file=None).LANCEDB_PATH == "/srv/lancedb" + + class TestPackageDir: def test_holds_the_shipped_data(self): assert paths.package_dir().name == "docsgpt" diff --git a/tests/test_cli.py b/tests/test_cli.py index 00d2c8bc..8574b5e1 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -43,7 +43,7 @@ class TestApi: run.assert_called_once_with() argv = sys.argv assert argv[0] == "gunicorn" and argv[-1] == "docsgpt.asgi:asgi_app" - assert argv[argv.index("--bind") + 1] == "0.0.0.0:8000" + assert argv[argv.index("--bind") + 1] == "127.0.0.1:8000" assert argv[argv.index("-w") + 1] == "2" assert argv[argv.index("-k") + 1] == "docsgpt.gunicorn_worker.BoundedDrainUvicornWorker" assert argv[argv.index("--config") + 1] == "python:docsgpt.gunicorn_conf" From f6f0d49452351e916fd3156054bbcf3993c60af9 Mon Sep 17 00:00:00 2001 From: Alex Date: Mon, 7 Sep 2026 16:09:52 +0100 Subject: [PATCH 3/4] docs(package): pipx CPU torch swap that works, and the local vector-store path note - pipx: replacing the CUDA torch inside the pipx environment needs pip's --force-reinstall (pipx inject, with or without --force, leaves a satisfied requirement alone), and --no-deps so torch's dependencies are not reinstalled from the PyTorch index, which carries older copies of them. - Upgrading: the embedded Milvus and LanceDB default paths now resolve under the checkout (or DOCSGPT_HOME) instead of the start directory; the note names the old location and the settings to pin. --- docs/content/Deploying/Pip-Install.mdx | 6 +++++- docs/content/upgrading.mdx | 10 ++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/docs/content/Deploying/Pip-Install.mdx b/docs/content/Deploying/Pip-Install.mdx index 84358c96..43dbf33c 100644 --- a/docs/content/Deploying/Pip-Install.mdx +++ b/docs/content/Deploying/Pip-Install.mdx @@ -41,7 +41,11 @@ pip install --index-url https://download.pytorch.org/whl/cpu torch torchvision pip install "docsgpt[docling]" ``` -`uv pip install` accepts the same two commands. With `pipx`, install `docsgpt[docling]` and then run `pipx inject --index-url https://download.pytorch.org/whl/cpu docsgpt torch torchvision` to swap the CUDA build for the CPU one. +`uv pip install` accepts the same two commands. With `pipx`, install `docsgpt[docling]` first, then replace the CUDA build inside its environment (`--no-deps` keeps pip from touching torch's dependencies, which the PyTorch index carries in older copies): + +```bash +pipx runpip docsgpt install --force-reinstall --no-deps --index-url https://download.pytorch.org/whl/cpu torch torchvision +``` ## Configure diff --git a/docs/content/upgrading.mdx b/docs/content/upgrading.mdx index abf71d2c..e00029a5 100644 --- a/docs/content/upgrading.mdx +++ b/docs/content/upgrading.mdx @@ -111,6 +111,16 @@ alias, so nothing breaks on upgrade, but update these before the alias goes: than it would have (a monthly sync by up to a month). Nothing to do. - Data directories do not move: the compose files keep your indexes, inputs and vectors under `application/` in the checkout, where they already are. +- The backend is also a package now (`pip install docsgpt`, see + [Install with pip](/Deploying/Pip-Install)). Runtime data lives in a data + home: `DOCSGPT_HOME`, else the checkout, else the directory the process + starts in. One consequence for a source checkout: the embedded Milvus + (`MILVUS_URI`) and LanceDB (`LANCEDB_PATH`) default paths now resolve under + the checkout instead of the start directory. If you use either store at its + default path and start DocsGPT from another directory, the old data is at + `/milvus_local.db` or `/data/lancedb`; + point the setting at it, or move it into the checkout. Faiss indexes and + uploads were already stored under the checkout and are unaffected. ## Check your version From 3036ece3fff27fa5d0668b06e54465bc8908f400 Mon Sep 17 00:00:00 2001 From: Alex Date: Mon, 7 Sep 2026 17:53:01 +0100 Subject: [PATCH 4/4] fix(cli): review pass on the docsgpt command - The image pins DOCSGPT_HOME=/app: it ships no checkout, so the data home no longer depends on the working directory. - api, worker, beat and migrate print the data home and env file they resolved, so an API and a worker started from different directories show it. - The worker passes -Q only when asked; a bare worker consumes every configured queue, which honours EMBEDDINGS_QUEUE and DOCUMENT_PARSE_QUEUE. - The worker runs through celery.start and returns its exit code; click usage errors print usage and exit 2 instead of a traceback. - Windows: solo pool and no embedded scheduler (celery rejects -B there), with a pointer to the new `docsgpt beat` command, which runs the scheduler on its own. - prefetch_models and verify_offline parse their arguments, so --help is help rather than a model name. - A DOCSGPT_ENV_FILE that is not a file raises instead of booting with defaults. --- docs/content/Deploying/Pip-Install.mdx | 4 +- docsgpt/Dockerfile | 5 ++- docsgpt/cli.py | 55 +++++++++++++++++++++----- docsgpt/core/paths.py | 5 ++- docsgpt/scripts/prefetch_models.py | 12 +++++- docsgpt/scripts/verify_offline.py | 10 ++++- tests/core/test_paths.py | 14 ++++++- tests/test_cli.py | 52 +++++++++++++++++++++--- 8 files changed, 134 insertions(+), 23 deletions(-) diff --git a/docs/content/Deploying/Pip-Install.mdx b/docs/content/Deploying/Pip-Install.mdx index 43dbf33c..8da950d3 100644 --- a/docs/content/Deploying/Pip-Install.mdx +++ b/docs/content/Deploying/Pip-Install.mdx @@ -80,7 +80,9 @@ docsgpt worker # in a second terminal: the Celery worker, with the scheduler The API applies pending migrations when it starts (`AUTO_MIGRATE`), so `docsgpt migrate` is the explicit step for deployments that want the schema in place before the first request or that run the API with a restricted database role. -The worker is not optional: query embedding runs on it, so search fails without one. `docsgpt worker --help` lists the queue, concurrency and pool options; `--no-beat` starts a worker without the scheduler when another worker already runs it. +Both commands print the data home they resolved on start-up. Run them from the same directory, or set `DOCSGPT_HOME` for both, so the worker finds the files the API stores and the API finds the indexes the worker builds. + +The worker is not optional: query embedding runs on it, so search fails without one. `docsgpt worker --help` lists the queue, concurrency and pool options; `--no-beat` starts a worker without the scheduler when another worker already runs it. On Windows the scheduler cannot be embedded, so run `docsgpt beat` in a third terminal. Other commands: diff --git a/docsgpt/Dockerfile b/docsgpt/Dockerfile index 02c36636..5c59d815 100644 --- a/docsgpt/Dockerfile +++ b/docsgpt/Dockerfile @@ -150,7 +150,10 @@ ENV FLASK_APP=app.py # Thread caps. onnxruntime (FastEmbed) ignores OMP_NUM_THREADS and sizes its # pool to the host's core count, which a CPU-limited container still reports; # EMBEDDINGS_THREADS pins it the way OMP_NUM_THREADS pinned torch before. -ENV MALLOC_ARENA_MAX=2 \ +# The image ships no checkout, so pin the data home (inputs, indexes, .env) +# instead of depending on the working directory. +ENV DOCSGPT_HOME=/app \ + MALLOC_ARENA_MAX=2 \ OMP_NUM_THREADS=4 \ MKL_NUM_THREADS=4 \ OPENBLAS_NUM_THREADS=4 \ diff --git a/docsgpt/cli.py b/docsgpt/cli.py index 046ed9f9..04fe1960 100644 --- a/docsgpt/cli.py +++ b/docsgpt/cli.py @@ -16,11 +16,18 @@ from docsgpt.version import __version__ DEFAULT_HOST = "127.0.0.1" DEFAULT_PORT = 7091 -DEFAULT_QUEUES = "docsgpt,parsing,embeddings" + + +def _announce_home() -> None: + """Say where runtime data and the env file come from; the API and the worker must agree.""" + from docsgpt.core.paths import env_file, home_dir + + print(f"docsgpt: data home {home_dir()} (env file {env_file()})", file=sys.stderr) def _api(args: argparse.Namespace) -> int: """Serve the ASGI app: gunicorn with the bounded-drain uvicorn worker, or uvicorn when reloading.""" + _announce_home() if args.reload or sys.platform == "win32": import uvicorn @@ -49,20 +56,43 @@ def _api(args: argparse.Namespace) -> int: return 0 -def _worker(args: argparse.Namespace) -> int: - """Run the Celery worker (with the beat scheduler unless ``--no-beat``).""" +def _celery(argv: list[str]) -> int: + """Run a celery subcommand on the app and return its exit code (usage errors print usage).""" + import click + from docsgpt.app import celery - argv = ["worker", "-l", args.loglevel, "-Q", args.queues] + try: + code = celery.start(argv) + except click.ClickException as exc: + exc.show() + return exc.exit_code + return int(code or 0) + + +def _worker(args: argparse.Namespace) -> int: + """Run the Celery worker, with the beat scheduler embedded unless ``--no-beat`` (or on Windows).""" + _announce_home() + windows = sys.platform == "win32" + argv = ["worker", "-l", args.loglevel] + if args.queues: + argv += ["-Q", args.queues] if args.concurrency: argv += ["--concurrency", str(args.concurrency)] - if args.beat: + if args.beat and windows: + print("docsgpt: the embedded scheduler is not available on Windows; run `docsgpt beat` separately.", file=sys.stderr) + elif args.beat: argv.append("-B") - pool = args.pool or ("solo" if sys.platform == "darwin" else None) + pool = args.pool or ("solo" if sys.platform in ("darwin", "win32") else None) if pool: argv += ["--pool", pool] - celery.worker_main(argv) - return 0 + return _celery(argv) + + +def _beat(args: argparse.Namespace) -> int: + """Run the beat scheduler on its own (Windows, or a worker started with ``--no-beat``).""" + _announce_home() + return _celery(["beat", "-l", args.loglevel]) def _migrate(args: argparse.Namespace) -> int: @@ -70,6 +100,7 @@ def _migrate(args: argparse.Namespace) -> int: from docsgpt.core.settings import settings from docsgpt.storage.db.bootstrap import ensure_database_ready + _announce_home() logging.basicConfig(level=logging.INFO, format="%(levelname)s %(name)s: %(message)s") if not settings.POSTGRES_URI: print("POSTGRES_URI is not set; nothing to migrate.", file=sys.stderr) @@ -112,13 +143,17 @@ def build_parser() -> argparse.ArgumentParser: api.set_defaults(func=_api) worker = commands.add_parser("worker", help="run the Celery worker (and the scheduler)") - worker.add_argument("-Q", "--queues", default=DEFAULT_QUEUES, help=f"queues to consume (default: {DEFAULT_QUEUES})") + worker.add_argument("-Q", "--queues", help="queues to consume (default: every configured queue)") worker.add_argument("--concurrency", type=int, help="worker processes (default: one per CPU)") - worker.add_argument("--pool", help="celery pool (default: prefork; solo on macOS)") + worker.add_argument("--pool", help="celery pool (default: prefork; solo on macOS and Windows)") worker.add_argument("-l", "--loglevel", default="INFO") worker.add_argument("--no-beat", dest="beat", action="store_false", help="do not embed the beat scheduler") worker.set_defaults(func=_worker) + beat = commands.add_parser("beat", help="run the beat scheduler on its own") + beat.add_argument("-l", "--loglevel", default="INFO") + beat.set_defaults(func=_beat) + migrate = commands.add_parser("migrate", help="create the database if needed and run the migrations") migrate.add_argument("--no-create", dest="create_db", action="store_false", help="fail instead of creating a missing database") migrate.set_defaults(func=_migrate) diff --git a/docsgpt/core/paths.py b/docsgpt/core/paths.py index 48296d21..ff7b46bd 100644 --- a/docsgpt/core/paths.py +++ b/docsgpt/core/paths.py @@ -43,5 +43,8 @@ def env_file() -> Path: """The ``.env`` file settings load: ``DOCSGPT_ENV_FILE``, else ``/.env``.""" configured = os.environ.get(ENV_FILE_ENV) if configured: - return Path(configured).expanduser() + path = Path(configured).expanduser() + if not path.is_file(): + raise FileNotFoundError(f"{ENV_FILE_ENV} is set to {path}, which is not a file") + return path return home_dir() / ".env" diff --git a/docsgpt/scripts/prefetch_models.py b/docsgpt/scripts/prefetch_models.py index 7462311e..cee140e4 100644 --- a/docsgpt/scripts/prefetch_models.py +++ b/docsgpt/scripts/prefetch_models.py @@ -97,11 +97,21 @@ def prefetch(names: Sequence[str], cache_dir: Optional[str] = None) -> List[str] return fetched +def _parse(argv: Optional[Sequence[str]], prog: str, description: str) -> list[str]: + import argparse + + parser = argparse.ArgumentParser(prog=prog, description=description) + parser.add_argument( + "models", nargs="*", help=f"embedding model names or aliases (default: {', '.join(DEFAULT_MODELS)})" + ) + return parser.parse_args(argv).models or list(DEFAULT_MODELS) + + def main(argv: Optional[Sequence[str]] = None) -> int: logging.basicConfig(level=logging.INFO, format="%(levelname)s %(message)s") import os - names = list(argv) if argv else list(DEFAULT_MODELS) + names = _parse(argv, "prefetch-models", "Download the embedding models and the tiktoken encodings into the local caches.") fetched = prefetch(names, os.environ.get("EMBEDDINGS_CACHE_DIR")) logger.info("Cached %d model(s): %s", len(fetched), ", ".join(fetched)) encodings = prefetch_tiktoken() diff --git a/docsgpt/scripts/verify_offline.py b/docsgpt/scripts/verify_offline.py index 95377ea7..9a6b27aa 100644 --- a/docsgpt/scripts/verify_offline.py +++ b/docsgpt/scripts/verify_offline.py @@ -138,8 +138,16 @@ def _docling_check() -> object: def main(argv: Optional[Sequence[str]] = None) -> int: + import argparse + logging.basicConfig(level=logging.WARNING, format="%(levelname)s %(message)s") - models: List[str] = list(argv) if argv else list(DEFAULT_MODELS) + parser = argparse.ArgumentParser( + prog="verify-offline", description="Check that the install serves its defaults without network access." + ) + parser.add_argument( + "models", nargs="*", help=f"embedding model names or aliases to check (default: {', '.join(DEFAULT_MODELS)})" + ) + models: List[str] = parser.parse_args(argv).models or list(DEFAULT_MODELS) if _network_reachable(): print("note network is reachable; run with --network none to prove the offline path") else: diff --git a/tests/core/test_paths.py b/tests/core/test_paths.py index f6cb7a0b..e625626b 100644 --- a/tests/core/test_paths.py +++ b/tests/core/test_paths.py @@ -2,6 +2,8 @@ from pathlib import Path +import pytest + from docsgpt.core import paths REPO_ROOT = Path(__file__).resolve().parents[2] @@ -32,8 +34,16 @@ class TestEnvFile: assert paths.env_file() == tmp_path.resolve() / ".env" def test_the_env_var_wins(self, monkeypatch, tmp_path): - monkeypatch.setenv(paths.ENV_FILE_ENV, str(tmp_path / "custom.env")) - assert paths.env_file() == tmp_path / "custom.env" + custom = tmp_path / "custom.env" + custom.write_text("LLM_NAME=x\n") + monkeypatch.setenv(paths.ENV_FILE_ENV, str(custom)) + assert paths.env_file() == custom + + def test_a_missing_configured_file_is_an_error(self, monkeypatch, tmp_path): + """Booting with every default because of a typo in the path must not be silent.""" + monkeypatch.setenv(paths.ENV_FILE_ENV, str(tmp_path / "nope.env")) + with pytest.raises(FileNotFoundError, match="DOCSGPT_ENV_FILE"): + paths.env_file() class TestSettingsFollowTheHome: diff --git a/tests/test_cli.py b/tests/test_cli.py index 8574b5e1..af60c7df 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -6,6 +6,7 @@ import types from pathlib import Path from unittest.mock import MagicMock +import click import pytest from docsgpt import cli @@ -34,7 +35,7 @@ class TestTopLevel: class TestApi: - def test_gunicorn_argv(self, monkeypatch): + def test_gunicorn_argv(self, monkeypatch, capsys): run = MagicMock() monkeypatch.setitem(sys.modules, "gunicorn.app.wsgiapp", types.SimpleNamespace(run=run)) monkeypatch.setattr(sys, "platform", "linux") @@ -47,6 +48,7 @@ class TestApi: assert argv[argv.index("-w") + 1] == "2" assert argv[argv.index("-k") + 1] == "docsgpt.gunicorn_worker.BoundedDrainUvicornWorker" assert argv[argv.index("--config") + 1] == "python:docsgpt.gunicorn_conf" + assert "data home" in capsys.readouterr().err def test_reload_uses_uvicorn(self, monkeypatch): uvicorn = types.SimpleNamespace(run=MagicMock()) @@ -57,31 +59,62 @@ class TestApi: class TestWorker: @staticmethod - def _celery(monkeypatch): + def _celery(monkeypatch, start=None): celery = MagicMock() + celery.start = start or MagicMock(return_value=0) monkeypatch.setitem(sys.modules, "docsgpt.app", types.SimpleNamespace(celery=celery)) return celery - def test_defaults(self, monkeypatch): + def test_defaults_consume_every_configured_queue(self, monkeypatch, capsys): celery = self._celery(monkeypatch) monkeypatch.setattr(sys, "platform", "linux") assert cli.main(["worker"]) == 0 - argv = celery.worker_main.call_args.args[0] + argv = celery.start.call_args.args[0] assert argv[:3] == ["worker", "-l", "INFO"] - assert argv[argv.index("-Q") + 1] == cli.DEFAULT_QUEUES + assert "-Q" not in argv, "a bare worker honours EMBEDDINGS_QUEUE and DOCUMENT_PARSE_QUEUE" assert "-B" in argv assert "--pool" not in argv + assert "data home" in capsys.readouterr().err def test_options_and_the_macos_solo_pool(self, monkeypatch): celery = self._celery(monkeypatch) monkeypatch.setattr(sys, "platform", "darwin") assert cli.main(["worker", "--no-beat", "-Q", "embeddings", "--concurrency", "2"]) == 0 - argv = celery.worker_main.call_args.args[0] + argv = celery.start.call_args.args[0] assert argv[argv.index("--pool") + 1] == "solo" assert "-B" not in argv assert argv[argv.index("-Q") + 1] == "embeddings" assert argv[argv.index("--concurrency") + 1] == "2" + def test_windows_runs_solo_without_the_embedded_scheduler(self, monkeypatch, capsys): + celery = self._celery(monkeypatch) + monkeypatch.setattr(sys, "platform", "win32") + assert cli.main(["worker"]) == 0 + argv = celery.start.call_args.args[0] + assert "-B" not in argv + assert argv[argv.index("--pool") + 1] == "solo" + assert "docsgpt beat" in capsys.readouterr().err + + def test_the_worker_exit_code_is_returned(self, monkeypatch): + self._celery(monkeypatch, start=MagicMock(return_value=1)) + monkeypatch.setattr(sys, "platform", "linux") + assert cli.main(["worker"]) == 1 + + def test_a_usage_error_prints_usage_instead_of_a_traceback(self, monkeypatch, capsys): + self._celery(monkeypatch, start=MagicMock(side_effect=click.UsageError("No such option: --bogus"))) + monkeypatch.setattr(sys, "platform", "linux") + assert cli.main(["worker"]) == 2 + assert "No such option" in capsys.readouterr().err + + +class TestBeat: + def test_runs_the_scheduler_alone(self, monkeypatch): + celery = MagicMock() + celery.start = MagicMock(return_value=0) + monkeypatch.setitem(sys.modules, "docsgpt.app", types.SimpleNamespace(celery=celery)) + assert cli.main(["beat", "-l", "DEBUG"]) == 0 + assert celery.start.call_args.args[0] == ["beat", "-l", "DEBUG"] + class TestMigrate: def test_runs_the_bootstrap(self, monkeypatch): @@ -109,3 +142,10 @@ class TestScripts: def test_the_script_exit_code_is_returned(self, monkeypatch): monkeypatch.setattr("docsgpt.scripts.verify_offline.main", MagicMock(return_value=3)) assert cli.main(["verify-offline"]) == 3 + + @pytest.mark.parametrize("script", ["prefetch-models", "verify-offline"]) + def test_help_is_help_not_a_model_name(self, script, capsys): + with pytest.raises(SystemExit) as exc: + cli.main([script, "--help"]) + assert exc.value.code == 0 + assert "models" in capsys.readouterr().out