diff --git a/.github/workflows/docker-image-verify.yml b/.github/workflows/docker-image-verify.yml index fec67707..f592febf 100644 --- a/.github/workflows/docker-image-verify.yml +++ b/.github/workflows/docker-image-verify.yml @@ -23,6 +23,10 @@ on: - 'frontend/**' - 'scripts/build_frontend.sh' - 'deployment/docker-compose-standalone.yaml' + - 'docsgpt/deploy/**' + - 'docsgpt/cli.py' + - 'docsgpt/core/paths.py' + - 'pyproject.toml' - '.github/workflows/docker-image-verify.yml' permissions: @@ -94,6 +98,43 @@ jobs: curl -fsS "$base/settings" | grep -q 'src="/config.js"' echo "API and UI served on $base" + - name: Set up uv + if: matrix.variant == '' + uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6.8.0 + with: + # No cache: a cache restored into a job that runs the built image is a poisoning risk. + enable-cache: false + + - name: docsgpt up runs the same image from the installed package + if: matrix.variant == '' + run: | + set -euo pipefail + # Same Compose project name as the step above; stop that stack first. + docker compose -f deployment/docker-compose-standalone.yaml down -v + uv build --wheel --out-dir "$RUNNER_TEMP/dist" + uv venv "$RUNNER_TEMP/venv" + uv pip install --python "$RUNNER_TEMP/venv/bin/python" "$RUNNER_TEMP"/dist/*.whl + docsgpt="$RUNNER_TEMP/venv/bin/docsgpt" + stack="$RUNNER_TEMP/stack" + "$docsgpt" up --yes --dir "$stack" --image-tag verify + "$docsgpt" status --dir "$stack" + curl -fsS http://127.0.0.1:7091/ | grep -q 'src="/config.js"' + # Each secret must appear exactly once with a value: a missing or empty one + # falls back to a default silently. + check_secrets() { + for key in POSTGRES_PASSWORD JWT_SECRET_KEY; do + [ "$(grep -Ec "^$key=.+$" "$stack/.env")" -eq 1 ] + done + } + check_secrets + secrets=$(grep -E '^(POSTGRES_PASSWORD|JWT_SECRET_KEY)=.+$' "$stack/.env" | sort) + # Running up again keeps them; a changed database password locks the stack out of its volume. + "$docsgpt" up --yes --dir "$stack" --image-tag verify + check_secrets + [ "$(grep -E '^(POSTGRES_PASSWORD|JWT_SECRET_KEY)=.+$' "$stack/.env" | sort)" = "$secrets" ] + "$docsgpt" uninstall --yes --purge --dir "$stack" + test ! -e "$stack" + - name: Stack logs if: failure() env: diff --git a/.github/workflows/package-build.yml b/.github/workflows/package-build.yml index 0be880db..23798bfe 100644 --- a/.github/workflows/package-build.yml +++ b/.github/workflows/package-build.yml @@ -65,6 +65,8 @@ jobs: names = set(zipfile.ZipFile(glob.glob("dist/*.whl")[0]).namelist()) for required in ( "docsgpt/cli.py", + "docsgpt/deploy/commands.py", + "docsgpt/deploy/docker-compose.yaml", "docsgpt/alembic.ini", "docsgpt/alembic/env.py", "docsgpt/alembic/script.py.mako", diff --git a/deployment/docker-compose-standalone.yaml b/deployment/docker-compose-standalone.yaml index ca69253b..52d38d06 100644 --- a/deployment/docker-compose-standalone.yaml +++ b/deployment/docker-compose-standalone.yaml @@ -9,10 +9,11 @@ # the API; without it every ingest fails with a 401 (setup.sh generates one). # # The backend image serves the web UI and the API on one port. Every release -# also attaches this file as an asset. Settings come from .env next to this -# file (any DocsGPT setting, VITE_* included; the compose-internal service URLs -# below take precedence). Data lives in named volumes, so `docker compose down` -# keeps it and `docker compose down -v` removes it. +# also attaches this file as an asset, and `docsgpt up` runs it from the Python +# package. Settings come from .env next to this file (any DocsGPT setting, +# VITE_* included; the compose-internal service URLs below take precedence). +# Data lives in named volumes, so `docker compose down` keeps it and +# `docker compose down -v` removes it. # # DOCSGPT_IMAGE_TAG release to run, e.g. 0.20.0 (default: latest release); # develop follows the main branch @@ -55,6 +56,10 @@ services: - CACHE_REDIS_URL=redis://redis:6379/2 - POSTGRES_URI=postgresql://docsgpt:${POSTGRES_PASSWORD:-docsgpt}@postgres:5432/docsgpt - EMBEDDINGS_NAME=${EMBEDDINGS_NAME:-ibm-granite/granite-embedding-311m-multilingual-r2} + # A model server on the host (Ollama, vLLM, ...) is reachable as + # host.docker.internal on Linux too, as it is on Docker Desktop. + extra_hosts: + - "host.docker.internal:host-gateway" ports: - "${DOCSGPT_BIND:-127.0.0.1}:${DOCSGPT_PORT:-7091}:7091" volumes: @@ -84,6 +89,8 @@ services: - POSTGRES_URI=postgresql://docsgpt:${POSTGRES_PASSWORD:-docsgpt}@postgres:5432/docsgpt - API_URL=http://backend:7091 - EMBEDDINGS_NAME=${EMBEDDINGS_NAME:-ibm-granite/granite-embedding-311m-multilingual-r2} + extra_hosts: + - "host.docker.internal:host-gateway" volumes: - indexes:/app/indexes - inputs:/app/inputs diff --git a/docs/content/Deploying/Docker-Deploying.mdx b/docs/content/Deploying/Docker-Deploying.mdx index b9e24c90..fdc91229 100644 --- a/docs/content/Deploying/Docker-Deploying.mdx +++ b/docs/content/Deploying/Docker-Deploying.mdx @@ -17,6 +17,58 @@ Docker is the recommended method for deploying DocsGPT, providing a consistent a **Important Note for Windows Users:** Docker Desktop on Windows generally requires the WSL 2 backend to function correctly, especially when using features like host networking which are utilized in DocsGPT's Docker Compose setup. Ensure WSL 2 is enabled and configured in Docker Desktop settings. +## Run it with `docsgpt up` + +The `docsgpt` Python package can set up and run the stack described below for +you. It needs Docker with Compose 2.24 or newer, and Python 3.12 or newer (uv +installs one when it is missing): + +```bash +uv tool install docsgpt # or: pipx install docsgpt +docsgpt up +``` + +`docsgpt up` keeps the stack in `~/.docsgpt/server` (`/opt/docsgpt` when run as +root on Linux; `--dir` or `DOCSGPT_HOME` choose another folder): the Compose +file of the installed version, a `.env` with your settings and the generated +secrets, and `install.json`. Data lives in named Docker volumes. The first run +asks two questions: + +- **Who should reach DocsGPT:** only this computer; other machines on the + network (plain HTTP, with `AUTH_TYPE=simple_jwt` and an access token); or a + domain name with HTTPS (Caddy gets the certificate, access token as well). +- **Which model provider:** the DocsGPT public API (no key needed), OpenAI, + Anthropic, Google Gemini, OpenRouter, Groq, or an OpenAI-compatible server + such as Ollama or vLLM. + +Flags answer the same questions, for scripts and servers: + +```bash +docsgpt up --yes --domain docs.example.com --provider openai --api-key "$OPENAI_API_KEY" +``` + +Running `docsgpt up` again is safe: it keeps `.env` and the secrets and runs +the images of the installed package version. `docsgpt up --reconfigure` asks +the questions again. + +| Command | What it does | +| --- | --- | +| `docsgpt status` | Version, address, containers, and whether the API answers | +| `docsgpt logs [-f] [service]` | Container logs | +| `docsgpt token` | The access token, for installs reachable beyond this computer | +| `docsgpt open` | Open DocsGPT in the browser | +| `docsgpt env set KEY=VALUE` | Change a setting; `docsgpt up` applies it | +| `docsgpt upgrade` | Upgrade the package (for `uv tool` installs) and restart on the new version | +| `docsgpt down` | Stop the stack; data and settings stay | +| `docsgpt uninstall [--purge]` | Remove the containers; `--purge` also deletes the settings and all data | + +More `docsgpt up` options: `--port`, `--docling` (the image with the docling +parser engine and OCR), `--image-tag develop` (follow the `main` branch) and +`--adopt` (manage a stack you started from the standalone Compose file in +another folder; both use the same data volumes). For Ollama on the same +machine, use the base URL `http://host.docker.internal:11434/v1`; on Linux, +also make Ollama listen beyond localhost (`OLLAMA_HOST=0.0.0.0`). + ## Quickest Setup: Pre-built Images, No Checkout Every release publishes ready-to-run images to Docker Hub (`arc53/docsgpt`, diff --git a/docs/content/Deploying/Pip-Install.mdx b/docs/content/Deploying/Pip-Install.mdx index ffe92c1d..1b946687 100644 --- a/docs/content/Deploying/Pip-Install.mdx +++ b/docs/content/Deploying/Pip-Install.mdx @@ -13,6 +13,10 @@ DocsGPT is on PyPI as [`docsgpt`](https://pypi.org/project/docsgpt/): the API se `docsgpt api` serves the web UI on the same port as the API. Set `SERVE_UI=false` to run the API alone, for example behind the frontend Docker image or a UI you host yourself. + + With Docker available, the same package can run the whole stack for you, including Postgres and Redis: `docsgpt up`. See [Run it with `docsgpt up`](/Deploying/Docker-Deploying#run-it-with-docsgpt-up). + + ## Requirements - Python 3.12 or newer @@ -49,7 +53,11 @@ pipx runpip docsgpt install --force-reinstall --no-deps --index-url https://down ## Configure -DocsGPT keeps its runtime files in a **data home**: the `.env` file it reads settings from, uploaded files under `inputs/`, vector indexes under `indexes/` and downloaded embedding models under `models/`. The data home is the directory you run the commands from, or the directory `DOCSGPT_HOME` points to. `DOCSGPT_ENV_FILE` points at a `.env` kept somewhere else. Both variables must be set in the process environment, not in `.env`: they decide where `.env` is read from. +DocsGPT keeps its runtime files in a **data home**: the `.env` file it reads settings from, uploaded files under `inputs/`, vector indexes under `indexes/` and downloaded embedding models under `models/`. The data home is `~/.docsgpt/server` (`/opt/docsgpt` when you run as root on Linux), whatever directory you run the commands from. `DOCSGPT_HOME` moves it, and `DOCSGPT_ENV_FILE` points at a `.env` kept somewhere else. Both variables must be set in the process environment, not in `.env`: they decide where `.env` is read from. In a source checkout the data home is the checkout. + + + Up to 0.20 the data home of an installed package was the directory you ran the command from. If you kept `.env` and your data there, move them to `~/.docsgpt/server` or set `DOCSGPT_HOME` to that directory. `docsgpt api` and `docsgpt worker` point out a `.env` in the working directory that they no longer read. + Create a `.env` in the data home. The minimum for a hosted LLM: @@ -80,7 +88,7 @@ docsgpt worker # in a second terminal: the Celery worker, with the scheduler The API applies pending migrations when it starts (`AUTO_MIGRATE`), so `docsgpt migrate` is the explicit step for deployments that want the schema in place before the first request or that run the API with a restricted database role. -Both commands print the data home they resolved on start-up. Run them from the same directory, or set `DOCSGPT_HOME` for both, so the worker finds the files the API stores and the API finds the indexes the worker builds. +Both commands print the data home they resolved on start-up. They share it as long as `DOCSGPT_HOME` is the same for both (or unset), so the worker finds the files the API stores and the API finds the indexes the worker builds. The worker is not optional: query embedding runs on it, so search fails without one. `docsgpt worker --help` lists the queue, concurrency and pool options; `--no-beat` starts a worker without the scheduler when another worker already runs it. On Windows the scheduler cannot be embedded, so run `docsgpt beat` in a third terminal. diff --git a/docs/content/changelog.mdx b/docs/content/changelog.mdx index 0aa9324a..a655b79a 100644 --- a/docs/content/changelog.mdx +++ b/docs/content/changelog.mdx @@ -13,6 +13,22 @@ request, and [Upgrading](/upgrading) covers the steps an existing deployment has ## Unreleased +### `docsgpt up` runs DocsGPT on Docker + +The Python package now sets up and runs the Docker stack: `uv tool install docsgpt`, then +`docsgpt up`. The first run asks who should reach DocsGPT (this computer, the network with an +access token, or a domain with HTTPS) and which model provider to use, writes the settings and +secrets to `~/.docsgpt/server/.env`, and starts the images of the installed version. `docsgpt status`, +`logs`, `token`, `upgrade`, `down` and `uninstall` manage it afterwards. See +[Run it with `docsgpt up`](/Deploying/Docker-Deploying#run-it-with-docsgpt-up). + +### An installed package keeps its data in `~/.docsgpt/server` + +Outside a source checkout, the data home (`.env`, uploads, indexes, models) was the directory +the command ran from, so starting `docsgpt api` from another folder silently used other +settings. It is now `~/.docsgpt/server`, or `/opt/docsgpt` for root on Linux; `DOCSGPT_HOME` +still overrides it. See [Upgrading](/upgrading#pip-installs-data-home-moved). + ### The standalone Docker stack runs on one port The `arc53/docsgpt` image now serves the web UI next to the API, the way `docsgpt api` does from diff --git a/docs/content/upgrading.mdx b/docs/content/upgrading.mdx index 7df9a0e3..c5397390 100644 --- a/docs/content/upgrading.mdx +++ b/docs/content/upgrading.mdx @@ -11,6 +11,10 @@ import { Callout } from 'nextra/components' **Upgrading from 0.16.x?** User data moved from MongoDB to Postgres in 0.17.0. Follow the [Postgres Migration guide](/Deploying/Postgres-Migration) before running `docker compose pull` or `git pull` — existing deployments will not start cleanly without it. +## pip installs: data home moved + +An installed package (`pip install docsgpt`, pipx, `uv tool`) used to keep its data home, meaning `.env`, `inputs/`, `indexes/` and `models/`, in the directory you ran `docsgpt api` and `docsgpt worker` from. It is now `~/.docsgpt/server` (`/opt/docsgpt` for root on Linux). Either move those files there, or set `DOCSGPT_HOME` to the old directory in the environment of both commands. Both commands print the data home they use, and point out a `.env` in the working directory that they no longer read. Source checkouts and the Docker images are not affected. + ## Standalone Compose file: one port The standalone Compose file (`docker-compose-standalone.yaml`) no longer runs a frontend container. The backend image serves the web UI on port 7091, and the port is published on `127.0.0.1` unless you set `DOCSGPT_BIND`. After downloading the new file, start it with `--remove-orphans` to remove the old frontend container, then open port 7091 instead of 5173. If you opened DocsGPT from other machines, see [Upgrading from an earlier standalone file](/Deploying/Docker-Deploying#upgrading-from-an-earlier-standalone-file). The checkout Compose files and the Kubernetes manifests are unchanged. @@ -117,8 +121,9 @@ alias, so nothing breaks on upgrade, but update these before the alias goes: and vectors under `application/` in the checkout, where they already are. - The backend is also a package now (`pip install docsgpt`, see [Install with pip](/Deploying/Pip-Install)). Runtime data lives in a data - home: `DOCSGPT_HOME`, else the checkout, else the directory the process - starts in. One consequence for a source checkout: the embedded Milvus + home: `DOCSGPT_HOME`, else the checkout, else `~/.docsgpt/server` + (`/opt/docsgpt` for root on Linux; see + [pip installs: data home moved](#pip-installs-data-home-moved)). One consequence for a source checkout: the embedded Milvus (`MILVUS_URI`) and LanceDB (`LANCEDB_PATH`) default paths now resolve under the checkout instead of the start directory. If you use either store at its default path and start DocsGPT from another directory, the old data is at diff --git a/docsgpt/cli.py b/docsgpt/cli.py index e9d4d670..873cd099 100644 --- a/docsgpt/cli.py +++ b/docsgpt/cli.py @@ -1,4 +1,5 @@ -"""The ``docsgpt`` command: run the API, the worker and the maintenance scripts. +"""The ``docsgpt`` command: run the API, the worker and the maintenance scripts, +or run and manage DocsGPT on Docker (``docsgpt up``). Every subcommand imports what it needs when it runs, so ``docsgpt --help`` stays instant and does not touch the database. @@ -19,10 +20,24 @@ DEFAULT_PORT = 7091 def _announce_home() -> None: - """Say where runtime data and the env file come from; the API and the worker must agree.""" - from docsgpt.core.paths import env_file, home_dir + """Create the data home and say where data and the env file come from; the API and the worker must agree.""" + from pathlib import Path - print(f"docsgpt: data home {home_dir()} (env file {env_file()})", file=sys.stderr) + from docsgpt.core import paths + + home = paths.home_dir() + home.mkdir(parents=True, exist_ok=True) + env = paths.env_file() + print(f"docsgpt: data home {home} (env file {env})", file=sys.stderr) + # Up to 0.20 an installed package used the working directory as its home. + chosen = os.environ.get(paths.HOME_ENV) or os.environ.get(paths.ENV_FILE_ENV) or paths.checkout_root() + stray = Path.cwd() / ".env" + if not chosen and stray.is_file() and stray.resolve() != env.resolve(): + print( + f"docsgpt: {stray} is not used; settings come from {env}. " + f"Move the file there, or set DOCSGPT_HOME={Path.cwd()} to keep using this directory.", + file=sys.stderr, + ) def _gunicorn_options(host: str, port: int, workers: int) -> dict: @@ -139,6 +154,17 @@ def _migrate(args: argparse.Namespace) -> int: return 0 +def _deploy(name: str): + """A subcommand handler that imports ``docsgpt.deploy.commands`` only when it runs.""" + + def handler(args: argparse.Namespace, context=None) -> int: + from docsgpt.deploy import commands + + return getattr(commands, name)(args, context) + + return handler + + # Maintenance scripts keep their own argument parsers; the command hands # everything after the script name to them untouched (argparse would try to # interpret the options itself). @@ -155,11 +181,70 @@ def _run_script(module: str, argv: list[str]) -> int: return int(importlib.import_module(f"docsgpt.scripts.{module}").main(argv) or 0) +def _add_deploy_commands(commands) -> None: + """``docsgpt up`` and the commands that manage the Docker stack it runs.""" + from docsgpt.deploy.stack import EXPOSURES, PROVIDERS + + def stack_command(name: str, handler: str, help_text: str) -> argparse.ArgumentParser: + parser = commands.add_parser(name, help=help_text) + parser.add_argument("--dir", help="stack directory (default: DOCSGPT_HOME, else ~/.docsgpt/server)") + parser.set_defaults(func=_deploy(handler), deploy=True) + return parser + + up = stack_command("up", "up", "install or update DocsGPT on Docker and start it") + up.add_argument("--expose", choices=EXPOSURES, help="who can reach it: local (default), network or domain") + up.add_argument("--domain", help="public domain served over HTTPS by Caddy (implies --expose domain)") + up.add_argument("--port", type=int, help="host port for the UI and API (default: 7091)") + up.add_argument("--provider", choices=list(PROVIDERS), help="model provider (default: the DocsGPT public API)") + up.add_argument("--api-key", help="the provider's API key (or set DOCSGPT_API_KEY)") + up.add_argument("--model", help="model name (required for openai-compatible)") + up.add_argument("--base-url", help="base URL of an OpenAI-compatible server") + docling = up.add_mutually_exclusive_group() + docling.add_argument("--docling", dest="docling", action="store_const", const=True, + help="run the image with the docling parser engine and OCR (several GB larger)") + docling.add_argument("--no-docling", dest="docling", action="store_const", const=False, + help="go back to the default image") + up.set_defaults(docling=None) + up.add_argument("--image-tag", help="image tag to run instead of this package's version, e.g. develop") + up.add_argument("-y", "--yes", action="store_true", help="ask nothing: use the flags, then the defaults") + up.add_argument("--reconfigure", action="store_true", help="ask the setup questions again") + up.add_argument("--adopt", action="store_true", help="take over a DocsGPT stack started from another folder") + up.add_argument("--no-open", action="store_true", help="do not open the browser after the first install") + up.add_argument("--timeout", type=int, default=300, help="seconds to wait for the API to answer (default: 300)") + + stack_command("down", "down", "stop the Docker stack (data and settings stay)") + stack_command("status", "status", "show the stack's version, address, containers and health") + + logs = stack_command("logs", "logs", "show the stack's logs") + logs.add_argument("-f", "--follow", action="store_true", help="keep printing new lines") + logs.add_argument("--tail", type=int, help="only the last N lines of each service") + logs.add_argument("services", nargs="*", help="services to show, e.g. backend worker") + + stack_command("token", "token", "print the access token (installs reachable beyond this computer)") + stack_command("open", "open_ui", "open DocsGPT in the browser") + + upgrade = stack_command("upgrade", "upgrade", "upgrade the package and restart the stack on the new version") + upgrade.add_argument("--version", help="version to install (default: the latest release)") + + uninstall = stack_command("uninstall", "uninstall", "remove the Docker stack") + uninstall.add_argument("-y", "--yes", action="store_true", help="do not ask for confirmation") + uninstall.add_argument("--purge", action="store_true", help="also delete the settings and all data") + + env = stack_command("env", "env", "show, get or set the stack's settings") + env_actions = env.add_subparsers(dest="env_action", metavar="") + get = env_actions.add_parser("get", help="print one setting") + get.add_argument("key") + set_ = env_actions.add_parser("set", help="set settings (KEY=VALUE ...); run `docsgpt up` to apply") + set_.add_argument("pairs", nargs="+", metavar="KEY=VALUE") + + def build_parser() -> argparse.ArgumentParser: parser = argparse.ArgumentParser(prog="docsgpt", description="DocsGPT: private AI for agents, assistants and search.") parser.add_argument("--version", action="version", version=f"docsgpt {__version__}") commands = parser.add_subparsers(dest="command", metavar="") + _add_deploy_commands(commands) + api = commands.add_parser("api", help="serve the HTTP API") api.add_argument("--host", default=DEFAULT_HOST, help="interface to listen on (default: localhost; 0.0.0.0 for all)") api.add_argument("--port", type=int, default=DEFAULT_PORT) @@ -198,7 +283,18 @@ def main(argv: Optional[Sequence[str]] = None) -> int: if not args.command: parser.print_help() return 2 - return args.func(args) + if not getattr(args, "deploy", False): + return args.func(args) + from docsgpt.deploy.docker import DeployError + + try: + return args.func(args) + except DeployError as exc: + print(f"docsgpt: {exc}", file=sys.stderr) + return 1 + except KeyboardInterrupt: + print(file=sys.stderr) + return 130 if __name__ == "__main__": diff --git a/docsgpt/core/paths.py b/docsgpt/core/paths.py index ff7b46bd..b8e2ec77 100644 --- a/docsgpt/core/paths.py +++ b/docsgpt/core/paths.py @@ -5,15 +5,18 @@ ships with it (prompts, model catalogs, migrations). A checkout, when the package is imported from one, is the directory holding ``pyproject.toml``. The data home is where runtime data lives: ``.env``, ``inputs``, ``indexes``. -The home is ``DOCSGPT_HOME`` when set, else the checkout, else the current -directory. That keeps a source checkout and the Docker image (which runs from -``/app`` with the package beside it) behaving as before, and gives a -``pip install docsgpt`` user a home that is not ``site-packages``. +The home is ``DOCSGPT_HOME`` when set, else the checkout, else the default +home (``~/.docsgpt/server``, or ``/opt/docsgpt`` for root on Linux). That keeps +a source checkout and the Docker image (which pins ``DOCSGPT_HOME=/app``) +behaving as before, and gives an installed package one home no matter which +directory the command runs from. ``docsgpt up`` keeps its stack in the same +place. """ from __future__ import annotations import os +import sys from pathlib import Path HOME_ENV = "DOCSGPT_HOME" @@ -31,12 +34,19 @@ def checkout_root() -> Path | None: return root if (root / "pyproject.toml").is_file() else None +def default_home() -> Path: + """The home outside a checkout: ``/opt/docsgpt`` for root on Linux, else ``~/.docsgpt/server``.""" + if sys.platform.startswith("linux") and hasattr(os, "geteuid") and os.geteuid() == 0: + return Path("/opt/docsgpt") + return Path.home() / ".docsgpt" / "server" + + def home_dir() -> Path: - """Directory for runtime data: ``DOCSGPT_HOME``, else the checkout, else cwd.""" + """Directory for runtime data: ``DOCSGPT_HOME``, else the checkout, else the default home.""" configured = os.environ.get(HOME_ENV) if configured: return Path(configured).expanduser().resolve() - return checkout_root() or Path.cwd() + return checkout_root() or default_home() def env_file() -> Path: diff --git a/docsgpt/deploy/__init__.py b/docsgpt/deploy/__init__.py new file mode 100644 index 00000000..18c30821 --- /dev/null +++ b/docsgpt/deploy/__init__.py @@ -0,0 +1,5 @@ +"""Run DocsGPT on Docker from the installed package: ``docsgpt up`` and the commands that manage it. + +Nothing here imports the Flask app, Celery or the settings module, so these +commands start instantly and work before any configuration exists. +""" diff --git a/docsgpt/deploy/commands.py b/docsgpt/deploy/commands.py new file mode 100644 index 00000000..eaa38f44 --- /dev/null +++ b/docsgpt/deploy/commands.py @@ -0,0 +1,396 @@ +"""``docsgpt up`` and the commands that manage the stack it starts.""" + +from __future__ import annotations + +import getpass +import json +import os +import re +import shutil +import subprocess +import sys +import webbrowser +from collections.abc import Callable, Mapping +from dataclasses import dataclass +from datetime import datetime, timezone +from pathlib import Path +from typing import Any, Optional + +from docsgpt.deploy import envfile, stack +from docsgpt.deploy.docker import DeployError, Docker, lan_ip, wait_healthy + +PROJECT = "docsgpt" +DATABASE_VOLUME = f"{PROJECT}_postgres_data" +_MOVING_TAGS = ("latest", "develop") +_KEY = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$") +# Commands that stop or remove services name every profile, so Caddy goes too +# even when COMPOSE_PROFILES no longer enables it. +_EVERY_PROFILE = ("--profile", "https") + +EXPOSURE_CHOICES = [ + ("local", "Only this computer"), + ("network", "Other machines on the network (plain HTTP, access token)"), + ("domain", "A domain name with HTTPS (Caddy certificate, access token)"), +] + + +class Prompter: + """Questions on the terminal. The installer hands its terminal to ``docsgpt up``.""" + + def choose(self, question: str, options: list[tuple[str, str]], default: str) -> str: + keys = [key for key, _ in options] + print(question) + for number, (key, label) in enumerate(options, 1): + print(f" {number}) {label}{' (default)' if key == default else ''}") + while True: + answer = input(f"Choose 1-{len(options)} [{keys.index(default) + 1}]: ").strip() + if not answer: + return default + if answer.isdigit() and 1 <= int(answer) <= len(options): + return keys[int(answer) - 1] + if answer in keys: + return answer + print("Enter one of the numbers above.") + + def text(self, question: str, default: Optional[str] = None, secret: bool = False) -> str: + ask = getpass.getpass if secret else input + while True: + answer = ask(f"{question}{f' [{default}]' if default else ''}: ").strip() + if answer: + return answer + if default: + return default + + def confirm(self, question: str, default: bool = False) -> bool: + answer = input(f"{question} [{'Y/n' if default else 'y/N'}]: ").strip().lower() + return default if not answer else answer in ("y", "yes") + + +def detect_installer() -> str: + """How this ``docsgpt`` was installed: ``uv`` (uv tool), ``pipx`` or ``pip``.""" + prefix = Path(sys.prefix) + if (prefix / "uv-receipt.toml").is_file(): + return "uv" + if "pipx" in prefix.parts: + return "pipx" + return "pip" + + +def _run_command(args: list[str]) -> int: + try: + return subprocess.call(args) + except FileNotFoundError as exc: + raise DeployError(f"{args[0]} is not on PATH") from exc + + +def _exec_command(argv: list[str]) -> int: + """Replace this process with ``argv`` (the upgraded ``docsgpt``); Windows runs it and waits.""" + executable = shutil.which(argv[0]) or argv[0] + if sys.platform == "win32": + return subprocess.call([executable, *argv[1:]]) + os.execv(executable, [executable, *argv[1:]]) + return 0 # not reached + + +@dataclass +class Context: + """What the commands talk to; tests replace the parts that touch Docker, the terminal or the network.""" + + docker: Any + prompter: Any + interactive: bool + version: str + lan_ip: Callable[[], str] = lan_ip + wait: Callable[[str, float], bool] = wait_healthy + open_browser: Callable[[str], Any] = webbrowser.open + installer: Callable[[], str] = detect_installer + run: Callable[[list[str]], int] = _run_command + exec_up: Callable[[list[str]], int] = _exec_command + + @classmethod + def default(cls, args) -> "Context": + from docsgpt.version import __version__ + + interactive = sys.stdin.isatty() and not getattr(args, "yes", False) + return cls(docker=Docker(), prompter=Prompter(), interactive=interactive, version=__version__) + + +def _installed(directory: Path) -> Optional[dict[str, str]]: + """The stack's settings, or None (with a message) when there is no install in ``directory``.""" + if not (directory / stack.COMPOSE_FILE).is_file(): + print(f"No DocsGPT install in {directory}. Run `docsgpt up` first, or pass --dir.", file=sys.stderr) + return None + return envfile.read(directory / ".env") + + +def _current_provider(env: Mapping[str, str]) -> str: + name = env.get("LLM_PROVIDER", "docsgpt") + if name == "openai" and env.get("OPENAI_BASE_URL"): + return "openai-compatible" + return name if name in stack.PROVIDERS else "docsgpt" + + +def _check_other_stacks(args, context: Context, directory: Path) -> bool: + """True when containers of the project were started from another folder and ``up`` takes them over.""" + others = {path for path in context.docker.project_dirs(PROJECT) if path.resolve() != directory.resolve()} + if not others: + return False + if args.adopt: + return True + where = ", ".join(sorted(str(path) for path in others)) + question = ( + f"Docker already runs a DocsGPT stack started from {where}; it uses the same data volumes. " + f"Manage it from {directory} instead?" + ) + if context.interactive and context.prompter.confirm(question, default=False): + return True + raise DeployError( + f"Docker already runs a DocsGPT stack started from {where}. Stop it with `docker compose down` " + f"in that folder, or run again with --adopt to manage it from {directory}." + ) + + +def _choose_provider(args, context: Context, existing: Mapping[str, str], ask: bool): + name = args.provider + if name is None and ask: + name = context.prompter.choose("Which model provider?", list(stack.PROVIDERS.items()), _current_provider(existing)) + if name is None: + return None + api_key = args.api_key or os.environ.get("DOCSGPT_API_KEY") + model, base_url = args.model, args.base_url + if context.interactive: + if name == "openai-compatible": + base_url = base_url or context.prompter.text( + "Server base URL (Ollama on this machine: http://host.docker.internal:11434/v1)" + ) + model = model or context.prompter.text("Model name") + elif name != "docsgpt" and not api_key: + api_key = context.prompter.text(f"{stack.PROVIDERS[name]} API key", secret=True) + try: + return stack.provider_settings(name, api_key=api_key, model=model, base_url=base_url) + except ValueError as exc: + raise DeployError(str(exc)) from exc + + +def up(args, context: Optional[Context] = None) -> int: + """Install or update the stack in its directory and start it.""" + context = context or Context.default(args) + directory = stack.stack_dir(args.dir) + env_path = directory / ".env" + record_path = directory / stack.RECORD_FILE + existing = envfile.read(env_path) + configured = record_path.is_file() + + context.docker.preflight(context.interactive) + # Compose keeps containers whose configuration did not change, and with them the other + # folder's working-directory label; recreating them all makes the takeover complete. + recreate = ["--force-recreate"] if _check_other_stacks(args, context, directory) else [] + + ask = context.interactive and (not configured or args.reconfigure) + expose, domain = args.expose, args.domain + if ask and expose is None and domain is None: + expose = context.prompter.choose("Who should reach DocsGPT?", EXPOSURE_CHOICES, stack.exposure(existing)) + if expose == "domain" and not domain: + if not context.interactive: + raise DeployError("--expose domain needs --domain") + domain = context.prompter.text("Domain name (its DNS must point at this machine)", existing.get("DOCSGPT_DOMAIN")) + provider = _choose_provider(args, context, existing, ask) + + image_tag = args.image_tag or context.version + try: + updates = stack.plan( + existing, + image_tag=image_tag, + fresh_database=not context.docker.volume_exists(DATABASE_VOLUME), + expose=expose, + domain=domain, + port=args.port, + provider=provider, + docling=args.docling, + ) + except ValueError as exc: + raise DeployError(str(exc)) from exc + + directory.mkdir(parents=True, exist_ok=True) + shutil.copyfile(stack.compose_source(), directory / stack.COMPOSE_FILE) + envfile.update(env_path, updates) + env = envfile.read(env_path) + + if stack.exposure(existing) == "domain" and stack.exposure(env) != "domain": + # With the https profile off, `up --remove-orphans` would leave Caddy running on ports 80 and 443. + context.docker.compose(directory, *_EVERY_PROFILE, "rm", "--stop", "--force", "caddy", check=False) + if stack.exposure(env) == "network": + print( + "DocsGPT will listen on every interface over plain HTTP: its access token travels as " + "readable text. Use `docsgpt up --domain ` for HTTPS outside a trusted network.", + file=sys.stderr, + ) + up_args = ["up", "-d", "--remove-orphans", *recreate] + if image_tag in _MOVING_TAGS: + up_args += ["--pull", "always"] + print(f"Starting DocsGPT {image_tag} from {directory} ...") + context.docker.compose(directory, *up_args) + + health = stack.health_url(env) + print("Waiting for DocsGPT to answer (the first start also sets up the database) ...") + if not context.wait(health, args.timeout): + print( + f"DocsGPT did not answer at {health} within {args.timeout} seconds. " + "See what happened with `docsgpt logs backend`.", + file=sys.stderr, + ) + return 1 + + now = datetime.now(timezone.utc).isoformat(timespec="seconds") + record = json.loads(record_path.read_text(encoding="utf-8")) if configured else {"installed_at": now} + record.update(version=context.version, image_tag=image_tag, updated_at=now) + record_path.write_text(json.dumps(record, indent=2) + "\n", encoding="utf-8") + + address = stack.url(env, context.lan_ip()) + mode = stack.exposure(env) + print(f"\nDocsGPT is running at {address}") + if env.get("AUTH_TYPE") == "simple_jwt" and env.get("JWT_SECRET_KEY"): + print("Access token (the page asks for it; `docsgpt token` prints it again):") + print(f" {stack.simple_jwt_token(env['JWT_SECRET_KEY'])}") + if mode == "domain": + print("Caddy gets the certificate when it starts: DNS must point at this machine and ports 80 and 443 be open.") + print(f"Settings: {env_path} (change the model provider or access with `docsgpt up --reconfigure`)") + print("Manage it with: docsgpt status | logs | upgrade | down | uninstall") + if context.interactive and not configured and not args.no_open and mode == "local": + context.open_browser(address) + return 0 + + +def down(args, context: Optional[Context] = None) -> int: + """Stop the stack; data and settings stay.""" + context = context or Context.default(args) + directory = stack.stack_dir(args.dir) + if _installed(directory) is None: + return 1 + context.docker.compose(directory, *_EVERY_PROFILE, "down") + return 0 + + +def status(args, context: Optional[Context] = None) -> int: + """Version, address, containers and whether the API answers (exit 1 when it does not).""" + context = context or Context.default(args) + directory = stack.stack_dir(args.dir) + env = _installed(directory) + if env is None: + return 1 + tag = env.get("DOCSGPT_IMAGE_TAG", "unknown") + env.get("DOCSGPT_IMAGE_VARIANT", "") + print(f"DocsGPT {tag} in {directory}") + print(f"Address: {stack.url(env, context.lan_ip())} ({stack.exposure(env)})") + context.docker.compose(directory, "ps", check=False) + healthy = context.wait(stack.health_url(env), 0) + print("API: answering" if healthy else "API: not answering (see `docsgpt logs backend`)") + return 0 if healthy else 1 + + +def logs(args, context: Optional[Context] = None) -> int: + """``docker compose logs`` for the stack.""" + context = context or Context.default(args) + directory = stack.stack_dir(args.dir) + if _installed(directory) is None: + return 1 + options = (["--follow"] if args.follow else []) + (["--tail", str(args.tail)] if args.tail else []) + return context.docker.compose(directory, "logs", *options, *args.services, check=False).returncode + + +def token(args, context: Optional[Context] = None) -> int: + """Print the access token of a ``simple_jwt`` install.""" + directory = stack.stack_dir(args.dir) + env = _installed(directory) + if env is None: + return 1 + if env.get("AUTH_TYPE") != "simple_jwt" or not env.get("JWT_SECRET_KEY"): + print(f"This install has no access token (AUTH_TYPE={env.get('AUTH_TYPE') or 'none'}).", file=sys.stderr) + return 1 + print(stack.simple_jwt_token(env["JWT_SECRET_KEY"])) + return 0 + + +def open_ui(args, context: Optional[Context] = None) -> int: + """Open DocsGPT in the browser.""" + context = context or Context.default(args) + directory = stack.stack_dir(args.dir) + env = _installed(directory) + if env is None: + return 1 + address = stack.url(env, context.lan_ip()) + print(address) + context.open_browser(address) + return 0 + + +def env(args, context: Optional[Context] = None) -> int: + """Show where the settings are, or get and set them.""" + env_path = stack.stack_dir(args.dir) / ".env" + if args.env_action is None: + print(env_path) + return 0 + if args.env_action == "get": + values = envfile.read(env_path) + if args.key not in values: + print(f"{args.key} is not set in {env_path}", file=sys.stderr) + return 1 + print(values[args.key]) + return 0 + updates = {} + for pair in args.pairs: + key, separator, value = pair.partition("=") + if not separator or not _KEY.match(key): + raise DeployError(f"expected KEY=VALUE, got {pair!r}") + updates[key] = value + try: + envfile.update(env_path, updates) + except ValueError as exc: + raise DeployError(str(exc)) from exc + print(f"Saved to {env_path}. Run `docsgpt up` to apply.") + return 0 + + +def _uninstall_hint(installer: str) -> str: + return {"uv": "uv tool uninstall docsgpt", "pipx": "pipx uninstall docsgpt"}.get(installer, "pip uninstall docsgpt") + + +def upgrade(args, context: Optional[Context] = None) -> int: + """Upgrade the package, then run the new version's ``docsgpt up``.""" + context = context or Context.default(args) + spec = f"docsgpt=={args.version}" if args.version else "docsgpt" + installer = context.installer() + if installer == "uv": + if context.run(["uv", "tool", "install", "--force", spec]) != 0: + raise DeployError(f"uv could not install {spec}") + return context.exec_up(["docsgpt", "up", "--dir", str(stack.stack_dir(args.dir))]) + command = f"pipx install --force {spec}" if installer == "pipx" else f"pip install -U {spec}" + print(f"Upgrade the package with `{command}`, then run `docsgpt up` to move the stack to it.", file=sys.stderr) + return 1 + + +def uninstall(args, context: Optional[Context] = None) -> int: + """Remove the containers and the stack files; ``--purge`` also deletes settings and data.""" + context = context or Context.default(args) + directory = stack.stack_dir(args.dir) + if _installed(directory) is None: + return 1 + if args.purge: + what = "containers, settings and data (documents, conversations, the database)" + else: + what = "containers (the settings in .env and the data volumes are kept)" + if not args.yes: + if not context.interactive: + raise DeployError("uninstall needs --yes when there is no terminal to confirm on") + if not context.prompter.confirm(f"Remove the DocsGPT {what} in {directory}?", default=False): + print("Nothing removed.") + return 1 + context.docker.compose(directory, *_EVERY_PROFILE, "down", "--remove-orphans", *(["-v"] if args.purge else [])) + if args.purge: + shutil.rmtree(directory) + print(f"Removed DocsGPT and its data from {directory}.") + else: + for name in (stack.COMPOSE_FILE, stack.RECORD_FILE): + (directory / name).unlink(missing_ok=True) + print(f"Removed the containers. Settings stay in {directory / '.env'} and data in the Docker volumes.") + print(f"To remove the docsgpt command too: {_uninstall_hint(context.installer())}") + return 0 diff --git a/docsgpt/deploy/docker.py b/docsgpt/deploy/docker.py new file mode 100644 index 00000000..795aaeb9 --- /dev/null +++ b/docsgpt/deploy/docker.py @@ -0,0 +1,160 @@ +"""The docker CLI calls behind ``docsgpt up``.""" + +from __future__ import annotations + +import http.client +import re +import shutil +import socket +import subprocess +import sys +import time +import urllib.request +from collections.abc import Callable, Sequence +from pathlib import Path +from typing import Optional + +MIN_COMPOSE = (2, 24, 0) +DAEMON_START_SECONDS = 120 + + +class DeployError(Exception): + """A problem the user can act on; the command prints it without a traceback.""" + + +def run(args: Sequence[str], *, cwd: Optional[Path] = None, capture: bool = False, check: bool = True): + """Run a command, streaming its output unless ``capture``; with ``check`` a failure raises DeployError.""" + try: + result = subprocess.run(list(args), cwd=cwd, text=True, capture_output=capture, check=False) + except FileNotFoundError as exc: + raise DeployError(f"{args[0]} is not installed or not on PATH") from exc + if check and result.returncode != 0: + detail = (result.stderr or "").strip() if capture else "" + message = f"`{' '.join(args)}` failed with exit code {result.returncode}" + raise DeployError(f"{message}: {detail}" if detail else message) + return result + + +def _parse_version(text: str) -> Optional[tuple[int, ...]]: + match = re.search(r"(\d+)\.(\d+)\.(\d+)", text or "") + return tuple(int(part) for part in match.groups()) if match else None + + +class Docker: + """Docker and Docker Compose, through their command-line tools.""" + + def __init__( + self, + runner: Callable[..., subprocess.CompletedProcess] = run, + which: Callable[[str], Optional[str]] = shutil.which, + sleep: Callable[[float], None] = time.sleep, + platform: str = sys.platform, + ) -> None: + self._run = runner + self._which = which + self._sleep = sleep + self._platform = platform + + def preflight(self, interactive: bool = False) -> None: + """Make sure Docker is installed and running and Compose is new enough, starting Docker Desktop on macOS.""" + if not self._which("docker"): + raise DeployError("Docker is not installed. Get it from https://docs.docker.com/get-docker/ and run this again.") + info = self._run(["docker", "info"], capture=True, check=False) + if info.returncode != 0: + if "permission denied" in (info.stderr or "").lower(): + raise DeployError( + "Your user cannot use Docker (permission denied on its socket). Add it to the docker group " + "with `sudo usermod -aG docker $USER`, log out and back in, and run this again." + ) + self._start_daemon() + result = self._run(["docker", "compose", "version", "--short"], capture=True, check=False) + version = _parse_version(result.stdout) if result.returncode == 0 else None + if version is None: + raise DeployError( + "Docker Compose v2 is not available (`docker compose version` failed). " + "Install the Compose plugin: https://docs.docker.com/compose/install/" + ) + if version < MIN_COMPOSE: + found = ".".join(str(part) for part in version) + raise DeployError(f"Docker Compose {found} is too old; DocsGPT needs 2.24 or newer.") + + def daemon_running(self) -> bool: + return self._run(["docker", "info"], capture=True, check=False).returncode == 0 + + def _start_daemon(self) -> None: + if self._platform == "darwin": + print("Docker is not running; starting Docker Desktop ...", file=sys.stderr) + self._run(["open", "-a", "Docker"], check=False) + for _ in range(DAEMON_START_SECONDS // 2): + self._sleep(2) + if self.daemon_running(): + return + raise DeployError("Docker Desktop did not start within two minutes. Start it and run this again.") + if self._platform.startswith("linux"): + raise DeployError("Docker is not running. Start it with `sudo systemctl start docker` and run this again.") + raise DeployError("Docker is not running. Start Docker Desktop and run this again.") + + def compose(self, directory: Path, *args: str, capture: bool = False, check: bool = True): + """``docker compose `` in ``directory``, which holds the Compose file and its ``.env``.""" + return self._run(["docker", "compose", *args], cwd=directory, capture=capture, check=check) + + def volume_exists(self, name: str) -> bool: + return self._run(["docker", "volume", "inspect", name], capture=True, check=False).returncode == 0 + + def project_dirs(self, project: str) -> set[Path]: + """The folders containers of Compose project ``project`` were started from.""" + result = self._run( + [ + "docker", "ps", "-a", + "--filter", f"label=com.docker.compose.project={project}", + "--format", '{{.Label "com.docker.compose.project.working_dir"}}', + ], + capture=True, + check=False, + ) + if result.returncode != 0: + return set() + return {Path(line.strip()) for line in result.stdout.splitlines() if line.strip()} + + +def wait_healthy( + url: str, + timeout: float, + *, + opener: Callable = urllib.request.urlopen, + sleep: Callable[[float], None] = time.sleep, + clock: Callable[[], float] = time.monotonic, +) -> bool: + """Poll ``url`` until it answers 2xx (True) or ``timeout`` seconds pass (False); tries at least once. + + No request starts once the deadline is reached, and neither the pauses nor the + requests after the first run past it. + """ + deadline = clock() + timeout + request_timeout = min(5.0, timeout) if timeout > 0 else 5.0 + while True: + try: + with opener(url, timeout=request_timeout) as response: + if 200 <= response.status < 300: + return True + except (OSError, http.client.HTTPException): + # Not answering yet: refused, reset, timed out or a broken response. Keep polling. + pass + remaining = deadline - clock() + if remaining <= 0: + return False + sleep(min(2.0, remaining)) + remaining = deadline - clock() + if remaining <= 0: + return False + request_timeout = min(5.0, remaining) + + +def lan_ip() -> str: + """This machine's address on its network, or ``localhost``. No packet is sent.""" + with socket.socket(socket.AF_INET, socket.SOCK_DGRAM) as probe: + try: + probe.connect(("192.0.2.1", 80)) + return probe.getsockname()[0] + except OSError: + return "localhost" diff --git a/docsgpt/deploy/envfile.py b/docsgpt/deploy/envfile.py new file mode 100644 index 00000000..ff91e09b --- /dev/null +++ b/docsgpt/deploy/envfile.py @@ -0,0 +1,80 @@ +"""Read and update a ``.env`` file in place, keeping the lines the user wrote.""" + +from __future__ import annotations + +import os +import re +from collections.abc import Mapping +from pathlib import Path +from typing import Optional + +_ASSIGNMENT = re.compile(r"^\s*(?:export\s+)?([A-Za-z_][A-Za-z0-9_]*)\s*=(.*)$") +# Values written without quotes: nothing Compose would interpolate, strip or treat as a comment. +_PLAIN = re.compile(r"^[A-Za-z0-9_./:@+,=-]*$") + + +def _parse_value(raw: str) -> str: + """The value of one assignment, with Compose's quoting rules.""" + value = raw.strip() + if len(value) >= 2 and value[0] == value[-1] == "'": + return value[1:-1] + if len(value) >= 2 and value[0] == value[-1] == '"': + return re.sub(r'\\(["\\])', r"\1", value[1:-1]).replace("$$", "$") + return value.split(" #", 1)[0].rstrip() + + +def _format_value(value: str) -> str: + """``value`` quoted so it reads back unchanged, and so Compose passes it on unchanged.""" + if "\n" in value or "\r" in value: + raise ValueError("a .env value cannot contain a newline") + if _PLAIN.match(value): + return value + if "'" not in value: + # Compose does not interpolate single-quoted values. + return f"'{value}'" + # Double quotes are interpolated: $$ is Compose's literal dollar. + return '"' + value.replace("\\", "\\\\").replace('"', '\\"').replace("$", "$$") + '"' + + +def read(path: Path) -> dict[str, str]: + """The assignments in ``path`` (empty when it does not exist); a repeated key keeps its last value.""" + path = Path(path) + if not path.is_file(): + return {} + values: dict[str, str] = {} + for line in path.read_text(encoding="utf-8").splitlines(): + match = _ASSIGNMENT.match(line) + if match: + values[match.group(1)] = _parse_value(match.group(2)) + return values + + +def update(path: Path, values: Mapping[str, Optional[str]]) -> None: + """Set each key in place (``None`` removes it), append new keys, and leave every other line alone. + + The file is left readable by its owner only, including one that existed with a wider mode: it holds secrets. + """ + formatted = {key: None if value is None else _format_value(value) for key, value in values.items()} + path = Path(path) + lines = path.read_text(encoding="utf-8").splitlines() if path.is_file() else [] + written: set[str] = set() + out: list[str] = [] + for line in lines: + match = _ASSIGNMENT.match(line) + key = match.group(1) if match else None + if key not in formatted: + out.append(line) + continue + if key not in written and formatted[key] is not None: + out.append(f"{key}={formatted[key]}") + written.add(key) + out.extend(f"{key}={value}" for key, value in formatted.items() if key not in written and value is not None) + + path.parent.mkdir(parents=True, exist_ok=True) + descriptor = os.open(path, os.O_WRONLY | os.O_CREAT, 0o600) + if hasattr(os, "fchmod"): + # The creation mode only applies to a new file; tighten an existing one before writing. + os.fchmod(descriptor, 0o600) + os.ftruncate(descriptor, 0) + with os.fdopen(descriptor, "w", encoding="utf-8") as handle: + handle.write("\n".join(out) + "\n" if out else "") diff --git a/docsgpt/deploy/stack.py b/docsgpt/deploy/stack.py new file mode 100644 index 00000000..bb577c56 --- /dev/null +++ b/docsgpt/deploy/stack.py @@ -0,0 +1,184 @@ +"""The stack ``docsgpt up`` runs: where it lives and what its ``.env`` holds.""" + +from __future__ import annotations + +import os +import secrets +from collections.abc import Callable, Mapping +from pathlib import Path +from typing import Optional + +from docsgpt.core import paths + +COMPOSE_FILE = "docker-compose.yaml" +RECORD_FILE = "install.json" +DEFAULT_PORT = 7091 +EXPOSURES = ("local", "network", "domain") + +PROVIDERS = { + "docsgpt": "DocsGPT public API (free, no key)", + "openai": "OpenAI", + "anthropic": "Anthropic", + "google": "Google Gemini", + "openrouter": "OpenRouter", + "groq": "Groq", + "openai-compatible": "OpenAI-compatible server (Ollama, vLLM, LM Studio, ...)", +} + +_LOCAL_BINDS = ("", "127.0.0.1", "localhost", "::1") +_ALL_INTERFACES = ("0.0.0.0", "::") + + +def compose_source() -> Path: + """The Compose file for this version: shipped in the wheel, or ``deployment/`` in a checkout.""" + packaged = paths.package_dir() / "deploy" / COMPOSE_FILE + if packaged.is_file(): + return packaged + root = paths.checkout_root() + if root is not None: + in_checkout = root / "deployment" / "docker-compose-standalone.yaml" + if in_checkout.is_file(): + return in_checkout + raise FileNotFoundError("the Compose file is missing from this docsgpt installation; reinstall the package") + + +def stack_dir(explicit: Optional[str]) -> Path: + """Where the stack lives: ``--dir``, else ``DOCSGPT_HOME``, else the default home (never a checkout).""" + if explicit: + return Path(explicit).expanduser().resolve() + configured = os.environ.get(paths.HOME_ENV) + if configured: + return Path(configured).expanduser().resolve() + return paths.default_home() + + +def _profiles(env: Mapping[str, str]) -> set[str]: + return {name.strip() for name in env.get("COMPOSE_PROFILES", "").split(",") if name.strip()} + + +def exposure(env: Mapping[str, str]) -> str: + """Who can reach the stack, read back from its ``.env``: local, network or domain.""" + if "https" in _profiles(env) and env.get("DOCSGPT_DOMAIN"): + return "domain" + if env.get("DOCSGPT_BIND", "") not in _LOCAL_BINDS: + return "network" + return "local" + + +def provider_settings( + name: str, + api_key: Optional[str] = None, + model: Optional[str] = None, + base_url: Optional[str] = None, +) -> dict[str, Optional[str]]: + """The model settings for a provider choice; keys another provider used are cleared.""" + if name not in PROVIDERS: + raise ValueError(f"unknown provider {name!r}; choose one of: {', '.join(PROVIDERS)}") + if name == "docsgpt": + return {"LLM_PROVIDER": "docsgpt", "API_KEY": None, "LLM_NAME": None, "OPENAI_BASE_URL": None} + if name == "openai-compatible": + if not base_url: + raise ValueError("an OpenAI-compatible server needs a base URL, e.g. http://host.docker.internal:11434/v1") + if not model: + raise ValueError("an OpenAI-compatible server needs a model name") + return { + "LLM_PROVIDER": "openai", + "API_KEY": api_key or "not-needed", + "LLM_NAME": model, + "OPENAI_BASE_URL": base_url, + } + if not api_key: + raise ValueError(f"{PROVIDERS[name]} needs an API key") + # Without LLM_NAME the model catalog picks the provider's first model. + return {"LLM_PROVIDER": name, "API_KEY": api_key, "LLM_NAME": model or None, "OPENAI_BASE_URL": None} + + +def plan( + existing: Mapping[str, str], + *, + image_tag: str, + fresh_database: bool, + expose: Optional[str] = None, + domain: Optional[str] = None, + port: Optional[int] = None, + provider: Optional[Mapping[str, Optional[str]]] = None, + docling: Optional[bool] = None, + secret: Optional[Callable[[], str]] = None, +) -> dict[str, Optional[str]]: + """The ``.env`` changes for an ``up``: only keys that change, ``None`` for a key to remove. + + Settings the user did not ask to change are left alone, secrets are generated + once, and the database password is only set for a database that does not exist + yet (Postgres reads it when the volume is created). + """ + secret = secret or (lambda: secrets.token_hex(32)) + wanted: dict[str, Optional[str]] = {"DOCSGPT_IMAGE_TAG": image_tag} + + if domain and expose is None: + expose = "domain" + if expose is None and "DOCSGPT_BIND" not in existing and "COMPOSE_PROFILES" not in existing: + expose = "local" + if expose == "local": + wanted.update(DOCSGPT_BIND="127.0.0.1", COMPOSE_PROFILES=None, DOCSGPT_DOMAIN=None) + elif expose == "network": + wanted.update(DOCSGPT_BIND="0.0.0.0", COMPOSE_PROFILES=None, DOCSGPT_DOMAIN=None) + elif expose == "domain": + if not domain: + raise ValueError("exposing DocsGPT on a domain needs the domain name") + wanted.update(DOCSGPT_BIND="127.0.0.1", COMPOSE_PROFILES="https", DOCSGPT_DOMAIN=domain) + elif expose is not None: + raise ValueError(f"unknown exposure {expose!r}; choose one of: {', '.join(EXPOSURES)}") + if expose in ("network", "domain") and not existing.get("AUTH_TYPE"): + wanted["AUTH_TYPE"] = "simple_jwt" + + for key in ("INTERNAL_KEY", "JWT_SECRET_KEY"): + if not existing.get(key): + wanted[key] = secret() + if not existing.get("POSTGRES_PASSWORD") and fresh_database: + wanted["POSTGRES_PASSWORD"] = secret() + if "VITE_API_STREAMING" not in existing: + wanted["VITE_API_STREAMING"] = "true" + if port is not None: + wanted["DOCSGPT_PORT"] = str(port) + if docling is not None: + wanted["DOCSGPT_IMAGE_VARIANT"] = "-docling" if docling else None + if provider is None and "LLM_PROVIDER" not in existing: + provider = provider_settings("docsgpt") + if provider: + wanted.update(provider) + + return { + key: value + for key, value in wanted.items() + if (value is None and key in existing) or (value is not None and existing.get(key) != value) + } + + +def _port(env: Mapping[str, str]) -> str: + return env.get("DOCSGPT_PORT") or str(DEFAULT_PORT) + + +def url(env: Mapping[str, str], lan_ip: str) -> str: + """The address to open DocsGPT at.""" + mode = exposure(env) + if mode == "domain": + return f"https://{env['DOCSGPT_DOMAIN']}" + if mode == "network": + bind = env.get("DOCSGPT_BIND", "") + host = lan_ip if bind in _ALL_INTERFACES else bind + return f"http://{host}:{_port(env)}" + return f"http://localhost:{_port(env)}" + + +def health_url(env: Mapping[str, str]) -> str: + """The API health check, reached from this machine whatever the exposure.""" + bind = env.get("DOCSGPT_BIND", "") + host = "127.0.0.1" if bind in _LOCAL_BINDS or bind in _ALL_INTERFACES else bind + return f"http://{host}:{_port(env)}/api/health" + + +def simple_jwt_token(secret_key: str) -> str: + """The token the API accepts under ``AUTH_TYPE=simple_jwt`` (it signs the same payload at start).""" + from jose import jwt + + return jwt.encode({"sub": "local"}, secret_key, algorithm="HS256") diff --git a/pyproject.toml b/pyproject.toml index 58493952..30f07df6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -194,8 +194,14 @@ exclude = [ "docsgpt/vectors/", ] +# `docsgpt up` runs the standalone Compose file of its own version. The file +# stays in deployment/ (the release asset and the curl instructions use it +# there); the wheel carries a copy inside the package. +[tool.hatch.build.targets.wheel.force-include] +"deployment/docker-compose-standalone.yaml" = "docsgpt/deploy/docker-compose.yaml" + [tool.hatch.build.targets.sdist] -include = ["/docsgpt"] +include = ["/docsgpt", "/deployment/docker-compose-standalone.yaml"] artifacts = ["docsgpt/static/**"] exclude = [ "docsgpt/Dockerfile", diff --git a/tests/core/test_paths.py b/tests/core/test_paths.py index a1c80acd..73abec86 100644 --- a/tests/core/test_paths.py +++ b/tests/core/test_paths.py @@ -20,11 +20,31 @@ class TestHomeDir: monkeypatch.setenv(paths.HOME_ENV, str(tmp_path)) assert paths.home_dir() == tmp_path.resolve() - def test_an_installed_package_falls_back_to_cwd(self, monkeypatch, tmp_path): + def test_an_installed_package_uses_the_default_home_not_cwd(self, monkeypatch, tmp_path): + """`docsgpt api` from any directory finds the same .env and data.""" monkeypatch.delenv(paths.HOME_ENV, raising=False) monkeypatch.setattr(paths, "checkout_root", lambda: None) + monkeypatch.setattr(paths, "default_home", lambda: tmp_path / "home") monkeypatch.chdir(tmp_path) - assert paths.home_dir() == Path.cwd() + assert paths.home_dir() == tmp_path / "home" + + +class TestDefaultHome: + def test_a_user_gets_a_folder_in_their_home(self, monkeypatch, tmp_path): + monkeypatch.setattr(paths.sys, "platform", "darwin") + monkeypatch.setattr(paths.Path, "home", classmethod(lambda cls: tmp_path)) + assert paths.default_home() == tmp_path / ".docsgpt" / "server" + + def test_root_on_linux_gets_opt(self, monkeypatch): + monkeypatch.setattr(paths.sys, "platform", "linux") + monkeypatch.setattr(paths.os, "geteuid", lambda: 0, raising=False) + assert paths.default_home() == Path("/opt/docsgpt") + + def test_a_normal_user_on_linux_stays_in_their_home(self, monkeypatch, tmp_path): + monkeypatch.setattr(paths.sys, "platform", "linux") + monkeypatch.setattr(paths.os, "geteuid", lambda: 1000, raising=False) + monkeypatch.setattr(paths.Path, "home", classmethod(lambda cls: tmp_path)) + assert paths.default_home() == tmp_path / ".docsgpt" / "server" class TestEnvFile: diff --git a/tests/deploy/__init__.py b/tests/deploy/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/deploy/test_commands.py b/tests/deploy/test_commands.py new file mode 100644 index 00000000..30faa350 --- /dev/null +++ b/tests/deploy/test_commands.py @@ -0,0 +1,337 @@ +"""`docsgpt up` and the commands that manage the stack, against a fake Docker.""" + +import json +import subprocess +import sys +from pathlib import Path + +import pytest + +from docsgpt import cli +from docsgpt.deploy import commands, envfile, stack +from docsgpt.deploy.docker import DeployError + +EVERY_PROFILE = ["--profile", "https"] + + +class FakeDocker: + def __init__(self, volumes=(), project_dirs=()): + self.volumes = set(volumes) + self.dirs = {Path(d) for d in project_dirs} + self.calls = [] + self.preflights = 0 + + def preflight(self, interactive=False): + self.preflights += 1 + + def compose(self, directory, *args, capture=False, check=True): + self.calls.append((Path(directory), list(args))) + if "down" in args and "-v" in args: + self.volumes.clear() + return subprocess.CompletedProcess(["docker", "compose", *args], 0, stdout="", stderr="") + + def volume_exists(self, name): + return name in self.volumes + + def project_dirs(self, project): + return set(self.dirs) + + +class FakePrompter: + def __init__(self, answers=()): + self.answers = list(answers) + self.questions = [] + + def _next(self, question): + self.questions.append(question) + if not self.answers: + raise AssertionError(f"unexpected question: {question}") + return self.answers.pop(0) + + def choose(self, question, options, default): + return self._next(question) + + def text(self, question, default=None, secret=False): + return self._next(question) + + def confirm(self, question, default=False): + return self._next(question) + + +def _context(docker=None, prompter=None, interactive=False, healthy=True, **overrides): + context = commands.Context( + docker=docker or FakeDocker(), + prompter=prompter or FakePrompter(), + interactive=interactive, + version="0.21.0", + lan_ip=lambda: "192.168.1.10", + wait=lambda url, timeout: healthy, + open_browser=lambda url: None, + ) + for key, value in overrides.items(): + setattr(context, key, value) + return context + + +def _run(argv, context): + args = cli.build_parser().parse_args(argv) + return args.func(args, context) + + +class TestUpFirstInstall: + def test_yes_installs_locally_with_the_public_api(self, tmp_path, capsys): + docker = FakeDocker() + assert _run(["up", "--yes", "--dir", str(tmp_path)], _context(docker)) == 0 + + assert (tmp_path / "docker-compose.yaml").read_text() == stack.compose_source().read_text() + env = envfile.read(tmp_path / ".env") + assert env["DOCSGPT_IMAGE_TAG"] == "0.21.0" + assert env["DOCSGPT_BIND"] == "127.0.0.1" + assert env["LLM_PROVIDER"] == "docsgpt" + assert env["INTERNAL_KEY"] and env["JWT_SECRET_KEY"] and env["POSTGRES_PASSWORD"] + assert docker.preflights == 1 + assert docker.calls == [(tmp_path, ["up", "-d", "--remove-orphans"])] + record = json.loads((tmp_path / "install.json").read_text()) + assert record["version"] == "0.21.0" + assert "http://localhost:7091" in capsys.readouterr().out + + def test_interactive_asks_who_reaches_it_and_which_model(self, tmp_path, capsys): + prompter = FakePrompter(["network", "anthropic", "sk-ant"]) + assert _run(["up", "--dir", str(tmp_path)], _context(prompter=prompter, interactive=True)) == 0 + env = envfile.read(tmp_path / ".env") + assert env["DOCSGPT_BIND"] == "0.0.0.0" + assert env["AUTH_TYPE"] == "simple_jwt" + assert env["LLM_PROVIDER"] == "anthropic" + assert env["API_KEY"] == "sk-ant" + out = capsys.readouterr().out + assert "http://192.168.1.10:7091" in out + assert stack.simple_jwt_token(env["JWT_SECRET_KEY"]) in out + + def test_flags_answer_the_questions(self, tmp_path): + argv = ["up", "--dir", str(tmp_path), "--domain", "docs.example.com", "--provider", "openai", "--api-key", "sk"] + assert _run(argv, _context(interactive=True)) == 0 + env = envfile.read(tmp_path / ".env") + assert env["COMPOSE_PROFILES"] == "https" + assert env["DOCSGPT_DOMAIN"] == "docs.example.com" + assert env["LLM_PROVIDER"] == "openai" + + def test_a_missing_api_key_is_an_error_without_a_terminal(self, tmp_path, monkeypatch): + monkeypatch.delenv("DOCSGPT_API_KEY", raising=False) + with pytest.raises(DeployError, match="API key"): + _run(["up", "--yes", "--dir", str(tmp_path), "--provider", "openai"], _context()) + + def test_the_api_key_can_come_from_the_environment(self, tmp_path, monkeypatch): + monkeypatch.setenv("DOCSGPT_API_KEY", "sk-env") + assert _run(["up", "--yes", "--dir", str(tmp_path), "--provider", "openai"], _context()) == 0 + assert envfile.read(tmp_path / ".env")["API_KEY"] == "sk-env" + + def test_an_existing_database_keeps_its_password(self, tmp_path): + docker = FakeDocker(volumes={"docsgpt_postgres_data"}) + assert _run(["up", "--yes", "--dir", str(tmp_path)], _context(docker)) == 0 + assert "POSTGRES_PASSWORD" not in envfile.read(tmp_path / ".env") + + def test_image_tag_and_docling(self, tmp_path): + argv = ["up", "--yes", "--dir", str(tmp_path), "--image-tag", "develop", "--docling"] + docker = FakeDocker() + assert _run(argv, _context(docker)) == 0 + env = envfile.read(tmp_path / ".env") + assert env["DOCSGPT_IMAGE_TAG"] == "develop" + assert env["DOCSGPT_IMAGE_VARIANT"] == "-docling" + # A moving tag is pulled every time, not only when missing. + assert (tmp_path, ["up", "-d", "--remove-orphans", "--pull", "always"]) in docker.calls + + def test_network_mode_warns_about_plain_http_before_starting(self, tmp_path, capsys): + """The token travels as readable text, so say so before the stack is up, not only after.""" + docker = FakeDocker() + start = docker.compose + seen = {} + + def recording(directory, *args, **kwargs): + # capsys hands over what was written so far, so the warning has to be in it already. + seen.setdefault("stderr", capsys.readouterr().err) + return start(directory, *args, **kwargs) + + docker.compose = recording + assert _run(["up", "--yes", "--dir", str(tmp_path), "--expose", "network"], _context(docker)) == 0 + assert docker.calls, "compose was called" + assert "plain HTTP" in seen["stderr"] + assert "--domain" in seen["stderr"] + + def test_a_local_install_does_not_warn(self, tmp_path, capsys): + assert _run(["up", "--yes", "--dir", str(tmp_path)], _context()) == 0 + assert "plain HTTP" not in capsys.readouterr().err + + def test_an_unhealthy_start_points_at_the_logs(self, tmp_path, capsys): + assert _run(["up", "--yes", "--dir", str(tmp_path)], _context(healthy=False)) == 1 + assert "docsgpt logs" in capsys.readouterr().err + assert not (tmp_path / "install.json").exists() + + +class TestUpAgain: + def test_keeps_the_settings_and_moves_the_version(self, tmp_path): + assert _run(["up", "--yes", "--dir", str(tmp_path)], _context()) == 0 + before = envfile.read(tmp_path / ".env") + envfile.update(tmp_path / ".env", {"CUSTOM": "mine"}) + + context = _context(interactive=True, docker=FakeDocker(volumes={"docsgpt_postgres_data"})) + context.version = "0.22.0" + assert _run(["up", "--dir", str(tmp_path)], context) == 0 + after = envfile.read(tmp_path / ".env") + assert after["DOCSGPT_IMAGE_TAG"] == "0.22.0" + assert after["CUSTOM"] == "mine" + for key in ("INTERNAL_KEY", "JWT_SECRET_KEY", "POSTGRES_PASSWORD"): + assert after[key] == before[key] + assert context.prompter.questions == [], "a configured install is not asked again" + + def test_leaving_the_domain_removes_caddy_before_starting(self, tmp_path): + """With the https profile off, `up --remove-orphans` alone would leave Caddy on ports 80 and 443.""" + assert _run(["up", "--yes", "--dir", str(tmp_path), "--domain", "docs.example.com"], _context()) == 0 + docker = FakeDocker(volumes={"docsgpt_postgres_data"}) + assert _run(["up", "--yes", "--dir", str(tmp_path), "--expose", "local"], _context(docker)) == 0 + assert docker.calls == [ + (tmp_path, [*EVERY_PROFILE, "rm", "--stop", "--force", "caddy"]), + (tmp_path, ["up", "-d", "--remove-orphans"]), + ] + + def test_staying_on_the_domain_leaves_caddy_alone(self, tmp_path): + assert _run(["up", "--yes", "--dir", str(tmp_path), "--domain", "docs.example.com"], _context()) == 0 + docker = FakeDocker(volumes={"docsgpt_postgres_data"}) + assert _run(["up", "--yes", "--dir", str(tmp_path)], _context(docker)) == 0 + assert docker.calls == [(tmp_path, ["up", "-d", "--remove-orphans"])] + + def test_another_projects_containers_need_consent(self, tmp_path): + docker = FakeDocker(project_dirs={"/srv/old-docsgpt"}) + with pytest.raises(DeployError, match="--adopt"): + _run(["up", "--yes", "--dir", str(tmp_path)], _context(docker)) + assert docker.calls == [] + assert _run(["up", "--yes", "--adopt", "--dir", str(tmp_path)], _context(docker)) == 0 + + def test_adopting_recreates_every_container(self, tmp_path): + """Compose keeps unchanged containers, and with them the old folder's label; the next up would ask again.""" + docker = FakeDocker(project_dirs={"/srv/old-docsgpt"}) + assert _run(["up", "--yes", "--adopt", "--dir", str(tmp_path)], _context(docker)) == 0 + assert docker.calls == [(tmp_path, ["up", "-d", "--remove-orphans", "--force-recreate"])] + + def test_containers_from_this_folder_are_not_recreated(self, tmp_path): + docker = FakeDocker(project_dirs={str(tmp_path)}) + assert _run(["up", "--yes", "--dir", str(tmp_path)], _context(docker)) == 0 + assert docker.calls == [(tmp_path, ["up", "-d", "--remove-orphans"])] + + def test_consent_can_be_given_at_the_prompt(self, tmp_path): + docker = FakeDocker(project_dirs={"/srv/old-docsgpt"}) + prompter = FakePrompter([True, "local", "docsgpt"]) + assert _run(["up", "--dir", str(tmp_path)], _context(docker, prompter, interactive=True)) == 0 + + +class TestManage: + @staticmethod + def _installed(tmp_path, *extra): + assert _run(["up", "--yes", "--dir", str(tmp_path), *extra], _context()) == 0 + + def test_down_includes_caddy(self, tmp_path): + self._installed(tmp_path) + docker = FakeDocker() + assert _run(["down", "--dir", str(tmp_path)], _context(docker)) == 0 + assert docker.calls == [(tmp_path, [*EVERY_PROFILE, "down"])] + + def test_commands_on_a_missing_install(self, tmp_path, capsys): + assert _run(["status", "--dir", str(tmp_path)], _context()) == 1 + assert "docsgpt up" in capsys.readouterr().err + + def test_status(self, tmp_path, capsys): + self._installed(tmp_path) + docker = FakeDocker() + assert _run(["status", "--dir", str(tmp_path)], _context(docker)) == 0 + out = capsys.readouterr().out + assert "0.21.0" in out and "http://localhost:7091" in out + assert (tmp_path, ["ps"]) in docker.calls + + def test_logs_pass_through(self, tmp_path): + self._installed(tmp_path) + docker = FakeDocker() + assert _run(["logs", "--dir", str(tmp_path), "-f", "--tail", "50", "backend"], _context(docker)) == 0 + assert docker.calls == [(tmp_path, ["logs", "--follow", "--tail", "50", "backend"])] + + def test_token(self, tmp_path, capsys): + self._installed(tmp_path, "--expose", "network") + capsys.readouterr() + assert _run(["token", "--dir", str(tmp_path)], _context()) == 0 + secret = envfile.read(tmp_path / ".env")["JWT_SECRET_KEY"] + assert capsys.readouterr().out.strip() == stack.simple_jwt_token(secret) + + def test_no_token_without_simple_jwt(self, tmp_path, capsys): + self._installed(tmp_path) + assert _run(["token", "--dir", str(tmp_path)], _context()) == 1 + assert "AUTH_TYPE" in capsys.readouterr().err + + def test_env_get_and_set(self, tmp_path, capsys): + self._installed(tmp_path) + capsys.readouterr() + assert _run(["env", "--dir", str(tmp_path), "set", "LLM_NAME=gpt-5.5", "OCR_ENABLED=true"], _context()) == 0 + assert "docsgpt up" in capsys.readouterr().out + assert _run(["env", "--dir", str(tmp_path), "get", "LLM_NAME"], _context()) == 0 + assert capsys.readouterr().out.strip() == "gpt-5.5" + assert _run(["env", "--dir", str(tmp_path), "get", "NOPE"], _context()) == 1 + with pytest.raises(DeployError, match="KEY=VALUE"): + _run(["env", "--dir", str(tmp_path), "set", "oops"], _context()) + + def test_uninstall_keeps_data_and_settings(self, tmp_path, capsys): + self._installed(tmp_path) + docker = FakeDocker() + assert _run(["uninstall", "--yes", "--dir", str(tmp_path)], _context(docker, installer=lambda: "uv")) == 0 + assert docker.calls == [(tmp_path, [*EVERY_PROFILE, "down", "--remove-orphans"])] + assert (tmp_path / ".env").is_file(), "the database password lives there" + assert not (tmp_path / "docker-compose.yaml").exists() + assert not (tmp_path / "install.json").exists() + assert "uv tool uninstall docsgpt" in capsys.readouterr().out + + def test_uninstall_purge_removes_everything(self, tmp_path): + directory = tmp_path / "stack" + self._installed(directory) + docker = FakeDocker() + assert _run(["uninstall", "--yes", "--purge", "--dir", str(directory)], _context(docker)) == 0 + assert docker.calls == [(directory, [*EVERY_PROFILE, "down", "--remove-orphans", "-v"])] + assert not directory.exists() + + def test_uninstall_asks_first(self, tmp_path): + self._installed(tmp_path) + docker = FakeDocker() + prompter = FakePrompter([False]) + assert _run(["uninstall", "--dir", str(tmp_path)], _context(docker, prompter, interactive=True)) == 1 + assert docker.calls == [] + + +class TestUpgrade: + def test_a_uv_tool_install_upgrades_and_runs_up_again(self, tmp_path): + self_calls = [] + context = _context( + installer=lambda: "uv", + run=lambda args: self_calls.append(args) or 0, + exec_up=lambda argv: self_calls.append(["exec", *argv]) or 0, + ) + assert _run(["upgrade", "--dir", str(tmp_path), "--version", "0.22.0"], context) == 0 + assert self_calls[0] == ["uv", "tool", "install", "--force", "docsgpt==0.22.0"] + assert self_calls[1] == ["exec", "docsgpt", "up", "--dir", str(tmp_path)] + + def test_latest_when_no_version_is_given(self, tmp_path): + self_calls = [] + context = _context(installer=lambda: "uv", run=lambda args: self_calls.append(args) or 0, + exec_up=lambda argv: 0) + assert _run(["upgrade", "--dir", str(tmp_path)], context) == 0 + assert self_calls[0] == ["uv", "tool", "install", "--force", "docsgpt"] + + def test_a_pip_install_is_told_what_to_run(self, tmp_path, capsys): + context = _context(installer=lambda: "pip", run=lambda args: pytest.fail("must not run")) + assert _run(["upgrade", "--dir", str(tmp_path)], context) == 1 + err = capsys.readouterr().err + assert "pip install -U docsgpt" in err and "docsgpt up" in err + + +class TestImports: + def test_the_deploy_commands_do_not_boot_the_app(self): + code = ( + "import sys, docsgpt.cli, docsgpt.deploy.commands; " + "loaded = {m for m in sys.modules if m in ('docsgpt.app', 'docsgpt.core.settings', 'celery', 'flask')}; " + "assert not loaded, loaded" + ) + subprocess.run([sys.executable, "-c", code], cwd=Path(__file__).resolve().parents[2], check=True) diff --git a/tests/deploy/test_docker.py b/tests/deploy/test_docker.py new file mode 100644 index 00000000..8e42556e --- /dev/null +++ b/tests/deploy/test_docker.py @@ -0,0 +1,172 @@ +"""The docker CLI calls behind `docsgpt up`, against a fake runner.""" + +import subprocess +from pathlib import Path + +import pytest + +from docsgpt.deploy import docker as docker_module +from docsgpt.deploy.docker import DeployError, Docker + + +class FakeRunner: + """Answers docker commands from a table of argv prefix -> (code, stdout[, stderr]) and records every call.""" + + def __init__(self, answers=None): + self.answers = list((answers or {}).items()) + self.calls = [] + + def __call__(self, args, *, cwd=None, capture=False, check=True): + self.calls.append((list(args), cwd)) + for prefix, answer in self.answers: + if list(args[: len(prefix)]) == list(prefix): + if callable(answer): + answer = answer() + code, out, err = (*answer, "")[:3] + if check and code != 0: + raise DeployError(f"{' '.join(args)} failed") + return subprocess.CompletedProcess(args, code, stdout=out, stderr=err) + return subprocess.CompletedProcess(args, 0, stdout="", stderr="") + + +def _docker(runner, platform="linux", which=lambda name: "/usr/bin/docker"): + return Docker(runner=runner, which=which, sleep=lambda seconds: None, platform=platform) + + +class TestPreflight: + def test_docker_missing(self): + with pytest.raises(DeployError, match="Docker is not installed"): + _docker(FakeRunner(), which=lambda name: None).preflight() + + def test_daemon_down_on_linux_says_how_to_start_it(self): + runner = FakeRunner({("docker", "info"): (1, "")}) + with pytest.raises(DeployError, match="systemctl start docker"): + _docker(runner).preflight() + + def test_no_permission_on_the_socket_says_how_to_get_it(self): + """A user outside the docker group is not told that Docker is down.""" + denied = "permission denied while trying to connect to the Docker daemon socket at unix:///var/run/docker.sock" + runner = FakeRunner({("docker", "info"): (1, "", denied)}) + with pytest.raises(DeployError, match="docker group"): + _docker(runner).preflight() + + def test_daemon_down_on_macos_starts_docker_desktop(self): + state = {"started": False} + + def info(): + return (0, "") if state["started"] else (1, "") + + def start(): + state["started"] = True + return (0, "") + + runner = FakeRunner( + { + ("docker", "info"): info, + ("open", "-a", "Docker"): start, + ("docker", "compose", "version"): (0, "v2.39.1-desktop.1\n"), + } + ) + _docker(runner, platform="darwin").preflight() + assert (["open", "-a", "Docker"], None) in runner.calls + + @pytest.mark.parametrize("version", ["v2.23.0", "2.20.2"]) + def test_compose_too_old(self, version): + runner = FakeRunner({("docker", "compose", "version"): (0, version + "\n")}) + with pytest.raises(DeployError, match="2.24"): + _docker(runner).preflight() + + def test_compose_missing(self): + runner = FakeRunner({("docker", "compose", "version"): (1, "")}) + with pytest.raises(DeployError, match="Docker Compose"): + _docker(runner).preflight() + + @pytest.mark.parametrize("version", ["v2.24.0", "2.39.1-desktop.1", "5.5.1"]) + def test_compose_new_enough(self, version): + runner = FakeRunner({("docker", "compose", "version"): (0, version + "\n")}) + _docker(runner).preflight() + + +class TestQueries: + def test_compose_runs_in_the_stack_directory(self, tmp_path): + runner = FakeRunner() + _docker(runner).compose(tmp_path, "up", "-d") + assert runner.calls == [(["docker", "compose", "up", "-d"], tmp_path)] + + def test_volume_exists(self): + runner = FakeRunner({("docker", "volume", "inspect", "docsgpt_postgres_data"): (0, "[]")}) + assert _docker(runner).volume_exists("docsgpt_postgres_data") + assert not _docker(FakeRunner({("docker", "volume"): (1, "")})).volume_exists("docsgpt_postgres_data") + + def test_project_directories(self): + runner = FakeRunner({("docker", "ps"): (0, "/srv/old\n/srv/old\n\n/home/me/.docsgpt/server\n")}) + assert _docker(runner).project_dirs("docsgpt") == {Path("/srv/old"), Path("/home/me/.docsgpt/server")} + args = runner.calls[0][0] + assert "label=com.docker.compose.project=docsgpt" in args + + +class TestWaitHealthy: + def test_succeeds_once_the_api_answers(self): + attempts = {"n": 0} + + def opener(url, timeout): + attempts["n"] += 1 + if attempts["n"] < 3: + raise OSError("connection refused") + return _Response(200) + + clock = iter(range(0, 1000, 2)) + assert docker_module.wait_healthy("http://127.0.0.1:7091/api/health", 60, opener=opener, + sleep=lambda s: None, clock=lambda: next(clock)) + assert attempts["n"] == 3 + + def test_no_request_starts_at_or_after_the_deadline(self): + """A short timeout must not be overrun by a sleep and one more five-second request.""" + now = {"t": 0.0} + starts = [] + + def opener(url, timeout): + starts.append((now["t"], timeout)) + now["t"] += 1 + raise OSError("connection refused") + + def sleep(seconds): + now["t"] += seconds + + assert not docker_module.wait_healthy("http://127.0.0.1:7091/api/health", 3, opener=opener, + sleep=sleep, clock=lambda: now["t"]) + assert starts, "the first attempt always runs" + for started, timeout in starts[1:]: + assert started < 3 + assert started + timeout <= 3 + assert now["t"] <= 3 + + def test_a_zero_timeout_still_tries_once(self): + calls = [] + + def opener(url, timeout): + calls.append(timeout) + return _Response(200) + + assert docker_module.wait_healthy("http://127.0.0.1:7091/api/health", 0, opener=opener, + sleep=lambda s: None, clock=lambda: 0.0) + assert len(calls) == 1 and calls[0] > 0 + + def test_gives_up_after_the_timeout(self): + def opener(url, timeout): + raise OSError("connection refused") + + clock = iter(range(0, 1000, 10)) + assert not docker_module.wait_healthy("http://127.0.0.1:7091/api/health", 30, opener=opener, + sleep=lambda s: None, clock=lambda: next(clock)) + + +class _Response: + def __init__(self, status): + self.status = status + + def __enter__(self): + return self + + def __exit__(self, *exc): + return False diff --git a/tests/deploy/test_envfile.py b/tests/deploy/test_envfile.py new file mode 100644 index 00000000..e292be1c --- /dev/null +++ b/tests/deploy/test_envfile.py @@ -0,0 +1,91 @@ +"""Reading and updating the stack's .env without losing what the user wrote.""" + +import os +import sys + +import pytest + +from docsgpt.deploy import envfile + + +class TestRead: + def test_comments_blanks_quotes_and_export(self, tmp_path): + path = tmp_path / ".env" + path.write_text( + "# settings\n" + "\n" + "LLM_PROVIDER=openai\n" + "export API_KEY='sk-123'\n" + 'LLM_NAME="gpt 5"\n' + "EMPTY=\n" + "not a line\n" + ) + assert envfile.read(path) == { + "LLM_PROVIDER": "openai", + "API_KEY": "sk-123", + "LLM_NAME": "gpt 5", + "EMPTY": "", + } + + def test_a_missing_file_is_empty(self, tmp_path): + assert envfile.read(tmp_path / ".env") == {} + + def test_the_last_duplicate_wins(self, tmp_path): + path = tmp_path / ".env" + path.write_text("A=1\nA=2\n") + assert envfile.read(path) == {"A": "2"} + + +class TestUpdate: + def test_changes_values_in_place_and_keeps_everything_else(self, tmp_path): + path = tmp_path / ".env" + path.write_text("# my settings\nLLM_PROVIDER=openai\n\nCUSTOM=keep me\nDOCSGPT_IMAGE_TAG=0.19.0\n") + envfile.update(path, {"DOCSGPT_IMAGE_TAG": "0.21.0"}) + assert path.read_text() == "# my settings\nLLM_PROVIDER=openai\n\nCUSTOM=keep me\nDOCSGPT_IMAGE_TAG=0.21.0\n" + + def test_appends_new_keys_and_removes_none(self, tmp_path): + path = tmp_path / ".env" + path.write_text("A=1\nB=2") + envfile.update(path, {"B": None, "C": "3"}) + assert path.read_text() == "A=1\nC=3\n" + + def test_duplicates_collapse_to_the_first_line(self, tmp_path): + path = tmp_path / ".env" + path.write_text("A=1\nX=y\nA=2\n") + envfile.update(path, {"A": "3"}) + assert path.read_text() == "A=3\nX=y\n" + + @pytest.mark.parametrize( + "value", + ["plain", "with space", "hash # inside", "it's", 'say "hi"', "back\\slash", "", "pa$w'rd", "a$$b'c", "$HOME"], + ) + def test_values_round_trip(self, tmp_path, value): + path = tmp_path / ".env" + envfile.update(path, {"VALUE": value}) + assert envfile.read(path)["VALUE"] == value + + def test_a_dollar_in_double_quotes_is_escaped_for_compose(self, tmp_path): + """Compose interpolates double-quoted .env values; $$ is its literal dollar.""" + path = tmp_path / ".env" + envfile.update(path, {"API_KEY": "pa$w'rd"}) + assert path.read_text() == "API_KEY=\"pa$$w'rd\"\n" + + @pytest.mark.skipif(sys.platform == "win32", reason="POSIX permissions") + def test_a_new_file_is_private(self, tmp_path): + path = tmp_path / "stack" / ".env" + envfile.update(path, {"JWT_SECRET_KEY": "s"}) + assert oct(os.stat(path).st_mode & 0o777) == oct(0o600) + + @pytest.mark.skipif(sys.platform == "win32", reason="POSIX permissions") + def test_an_existing_readable_file_is_made_private_before_writing(self, tmp_path): + """Secrets are rewritten into the file, so a permissive mode left from before must not stay.""" + path = tmp_path / ".env" + path.write_text("LLM_PROVIDER=openai\n") + os.chmod(path, 0o644) + envfile.update(path, {"JWT_SECRET_KEY": "s"}) + assert oct(os.stat(path).st_mode & 0o777) == oct(0o600) + assert envfile.read(path) == {"LLM_PROVIDER": "openai", "JWT_SECRET_KEY": "s"} + + def test_rejects_a_newline_in_a_value(self, tmp_path): + with pytest.raises(ValueError, match="newline"): + envfile.update(tmp_path / ".env", {"A": "1\n2"}) diff --git a/tests/deploy/test_stack.py b/tests/deploy/test_stack.py new file mode 100644 index 00000000..e32c2393 --- /dev/null +++ b/tests/deploy/test_stack.py @@ -0,0 +1,196 @@ +"""What `docsgpt up` writes to the stack's .env, and where the stack lives.""" + +from itertools import count +from pathlib import Path + +import pytest + +from docsgpt.core import paths +from docsgpt.deploy import stack + +REPO_ROOT = Path(__file__).resolve().parents[2] + + +def _secrets(): + numbers = count(1) + return lambda: f"secret{next(numbers)}" + + +def _first_install(**overrides): + options = {"image_tag": "0.21.0", "fresh_database": True, "secret": _secrets()} + options.update(overrides) + return stack.plan({}, **options) + + +class TestFirstInstall: + def test_defaults_are_local_with_the_public_api(self): + updates = _first_install() + assert updates["DOCSGPT_IMAGE_TAG"] == "0.21.0" + assert updates["DOCSGPT_BIND"] == "127.0.0.1" + assert updates["LLM_PROVIDER"] == "docsgpt" + assert updates["VITE_API_STREAMING"] == "true" + assert "AUTH_TYPE" not in updates + + def test_secrets_are_generated_once_each(self): + updates = _first_install() + generated = {updates["INTERNAL_KEY"], updates["JWT_SECRET_KEY"], updates["POSTGRES_PASSWORD"]} + assert len(generated) == 3 + + def test_an_existing_database_keeps_its_password(self): + """Postgres reads the password only when its volume is created.""" + updates = _first_install(fresh_database=False) + assert "POSTGRES_PASSWORD" not in updates + + +class TestRerun: + def test_secrets_and_settings_are_left_alone(self): + existing = { + "DOCSGPT_IMAGE_TAG": "0.20.0", + "INTERNAL_KEY": "k", + "JWT_SECRET_KEY": "j", + "POSTGRES_PASSWORD": "p", + "VITE_API_STREAMING": "true", + "LLM_PROVIDER": "anthropic", + "API_KEY": "sk", + "DOCSGPT_BIND": "0.0.0.0", + "AUTH_TYPE": "simple_jwt", + } + updates = stack.plan(existing, image_tag="0.21.0", fresh_database=False, secret=_secrets()) + assert updates == {"DOCSGPT_IMAGE_TAG": "0.21.0"} + + def test_a_missing_password_is_not_invented_for_an_existing_database(self): + updates = stack.plan({"INTERNAL_KEY": "k", "JWT_SECRET_KEY": "j"}, image_tag="x", fresh_database=False) + assert "POSTGRES_PASSWORD" not in updates + + +class TestExposure: + def test_network_publishes_everywhere_and_turns_on_auth(self): + updates = _first_install(expose="network") + assert updates["DOCSGPT_BIND"] == "0.0.0.0" + assert updates["AUTH_TYPE"] == "simple_jwt" + assert updates.get("COMPOSE_PROFILES") is None + + def test_domain_adds_caddy_and_auth_and_keeps_the_port_local(self): + updates = _first_install(expose="domain", domain="docs.example.com") + assert updates["COMPOSE_PROFILES"] == "https" + assert updates["DOCSGPT_DOMAIN"] == "docs.example.com" + assert updates["DOCSGPT_BIND"] == "127.0.0.1" + assert updates["AUTH_TYPE"] == "simple_jwt" + + def test_domain_needs_a_domain(self): + with pytest.raises(ValueError, match="domain"): + _first_install(expose="domain") + + def test_an_existing_auth_mode_is_not_downgraded(self): + updates = stack.plan({"AUTH_TYPE": "oidc"}, image_tag="x", expose="network", fresh_database=False) + assert "AUTH_TYPE" not in updates + + def test_back_to_local_removes_the_proxy(self): + existing = {"COMPOSE_PROFILES": "https", "DOCSGPT_DOMAIN": "docs.example.com", "DOCSGPT_BIND": "127.0.0.1"} + updates = stack.plan(existing, image_tag="x", expose="local", fresh_database=False) + assert updates["COMPOSE_PROFILES"] is None + assert updates["DOCSGPT_DOMAIN"] is None + + def test_port_and_docling(self): + updates = _first_install(port=8080, docling=True) + assert updates["DOCSGPT_PORT"] == "8080" + assert updates["DOCSGPT_IMAGE_VARIANT"] == "-docling" + assert stack.plan({"DOCSGPT_IMAGE_VARIANT": "-docling"}, image_tag="x", docling=False, fresh_database=False)[ + "DOCSGPT_IMAGE_VARIANT" + ] is None + + @pytest.mark.parametrize( + "env, mode", + [ + ({}, "local"), + ({"DOCSGPT_BIND": "0.0.0.0"}, "network"), + ({"COMPOSE_PROFILES": "https", "DOCSGPT_DOMAIN": "d.example.com"}, "domain"), + ], + ) + def test_the_mode_is_read_back_from_the_env(self, env, mode): + assert stack.exposure(env) == mode + + +class TestProviders: + def test_switching_provider_drops_the_old_keys(self): + existing = {"LLM_PROVIDER": "openai", "API_KEY": "sk", "LLM_NAME": "m", "OPENAI_BASE_URL": "http://x/v1"} + provider = stack.provider_settings("anthropic", api_key="ak") + updates = stack.plan(existing, image_tag="x", provider=provider, fresh_database=False) + assert updates["LLM_PROVIDER"] == "anthropic" + assert updates["API_KEY"] == "ak" + assert updates["LLM_NAME"] is None + assert updates["OPENAI_BASE_URL"] is None + + def test_the_public_api_needs_no_key(self): + assert stack.provider_settings("docsgpt") == { + "LLM_PROVIDER": "docsgpt", + "API_KEY": None, + "LLM_NAME": None, + "OPENAI_BASE_URL": None, + } + + def test_a_hosted_provider_needs_a_key(self): + with pytest.raises(ValueError, match="API key"): + stack.provider_settings("openai") + + def test_an_openai_compatible_server_needs_a_url_and_a_model(self): + with pytest.raises(ValueError, match="base URL"): + stack.provider_settings("openai-compatible", model="llama3") + settings = stack.provider_settings("openai-compatible", base_url="http://host.docker.internal:11434/v1", model="llama3") + assert settings == { + "LLM_PROVIDER": "openai", + "API_KEY": "not-needed", + "LLM_NAME": "llama3", + "OPENAI_BASE_URL": "http://host.docker.internal:11434/v1", + } + + def test_an_unknown_provider(self): + with pytest.raises(ValueError, match="unknown provider"): + stack.provider_settings("nope") + + +class TestUrls: + def test_local(self): + assert stack.url({"DOCSGPT_PORT": "8080"}, lan_ip="10.0.0.5") == "http://localhost:8080" + + def test_network_uses_the_machine_address(self): + assert stack.url({"DOCSGPT_BIND": "0.0.0.0"}, lan_ip="10.0.0.5") == "http://10.0.0.5:7091" + + def test_domain(self): + env = {"COMPOSE_PROFILES": "https", "DOCSGPT_DOMAIN": "docs.example.com"} + assert stack.url(env, lan_ip="10.0.0.5") == "https://docs.example.com" + + def test_health_is_always_checked_on_this_machine(self): + assert stack.health_url({"DOCSGPT_BIND": "0.0.0.0", "DOCSGPT_PORT": "9000"}) == "http://127.0.0.1:9000/api/health" + + +class TestToken: + def test_matches_what_the_api_prints(self): + """docsgpt/app.py signs {"sub": "local"} with JWT_SECRET_KEY for AUTH_TYPE=simple_jwt.""" + from jose import jwt + + token = stack.simple_jwt_token("s3cret") + assert token == jwt.encode({"sub": "local"}, "s3cret", algorithm="HS256") + assert jwt.decode(token, "s3cret", algorithms=["HS256"]) == {"sub": "local"} + + +class TestLocations: + def test_the_stack_dir_is_never_the_checkout(self, monkeypatch, tmp_path): + monkeypatch.delenv(paths.HOME_ENV, raising=False) + monkeypatch.setattr(paths, "default_home", lambda: tmp_path / "home") + assert stack.stack_dir(None) == tmp_path / "home" + + def test_docsgpt_home_and_then_an_explicit_dir_win(self, monkeypatch, tmp_path): + monkeypatch.setenv(paths.HOME_ENV, str(tmp_path / "env-home")) + assert stack.stack_dir(None) == (tmp_path / "env-home").resolve() + assert stack.stack_dir(str(tmp_path / "flag")) == (tmp_path / "flag").resolve() + + def test_a_checkout_uses_the_deployment_compose_file(self): + assert stack.compose_source() == REPO_ROOT / "deployment" / "docker-compose-standalone.yaml" + + def test_the_packaged_compose_file_wins(self, monkeypatch, tmp_path): + packaged = tmp_path / "docsgpt" / "deploy" / "docker-compose.yaml" + packaged.parent.mkdir(parents=True) + packaged.write_text("name: docsgpt\n") + monkeypatch.setattr(paths, "package_dir", lambda: tmp_path / "docsgpt") + assert stack.compose_source() == packaged diff --git a/tests/test_cli.py b/tests/test_cli.py index c8f934ce..452f5598 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -34,6 +34,57 @@ class TestTopLevel: subprocess.run([sys.executable, "-c", code], cwd=Path(__file__).resolve().parents[1], check=True) +class TestHome: + @staticmethod + def _installed(monkeypatch, tmp_path): + """An installed package (no checkout) with the default home under tmp_path.""" + from docsgpt.core import paths + + monkeypatch.delenv(paths.HOME_ENV, raising=False) + monkeypatch.delenv(paths.ENV_FILE_ENV, raising=False) + monkeypatch.setattr(paths, "checkout_root", lambda: None) + monkeypatch.setattr(paths, "default_home", lambda: tmp_path / "home") + return tmp_path / "home" + + def test_the_home_is_created_and_announced(self, monkeypatch, tmp_path, capsys): + home = self._installed(monkeypatch, tmp_path) + monkeypatch.chdir(tmp_path) + cli._announce_home() + assert home.is_dir() + assert f"data home {home}" in capsys.readouterr().err + + def test_an_env_file_left_in_the_working_directory_is_pointed_out(self, monkeypatch, tmp_path, capsys): + """Up to 0.20 an installed package read .env from the working directory.""" + home = self._installed(monkeypatch, tmp_path) + work = tmp_path / "work" + work.mkdir() + (work / ".env").write_text("LLM_PROVIDER=openai\n") + monkeypatch.chdir(work) + cli._announce_home() + err = capsys.readouterr().err + assert f"{work / '.env'} is not used" in err + assert f"DOCSGPT_HOME={work}" in err + assert str(home) in err + + def test_no_warning_when_the_home_is_chosen_explicitly(self, monkeypatch, tmp_path, capsys): + from docsgpt.core import paths + + self._installed(monkeypatch, tmp_path) + (tmp_path / ".env").write_text("LLM_PROVIDER=openai\n") + monkeypatch.setenv(paths.HOME_ENV, str(tmp_path / "elsewhere")) + monkeypatch.chdir(tmp_path) + cli._announce_home() + assert "is not used" not in capsys.readouterr().err + + def test_no_warning_when_the_working_directory_is_the_home(self, monkeypatch, tmp_path, capsys): + home = self._installed(monkeypatch, tmp_path) + home.mkdir() + (home / ".env").write_text("LLM_PROVIDER=openai\n") + monkeypatch.chdir(home) + cli._announce_home() + assert "is not used" not in capsys.readouterr().err + + class TestApi: def test_gunicorn_runs_with_the_image_settings_and_leaves_argv_alone(self, monkeypatch, capsys): application = MagicMock()