diff --git a/docs/content/Deploying/Docker-Deploying.mdx b/docs/content/Deploying/Docker-Deploying.mdx index b4c5f87f..ac835442 100644 --- a/docs/content/Deploying/Docker-Deploying.mdx +++ b/docs/content/Deploying/Docker-Deploying.mdx @@ -79,6 +79,42 @@ the questions again. | `docsgpt down` | Stop the stack; data and settings stay | | `docsgpt uninstall [--purge]` | Remove the containers; `--purge` also deletes the settings and all data | +### Backups + +`docsgpt backup` writes one archive holding a dump of the database and a tar of +each data volume (`indexes`, `inputs`, `vectors`): + +```bash +docsgpt backup # into /backups +docsgpt backup --out /mnt/backups # somewhere else, e.g. a mounted disk +``` + +While the archive is made, the backend and the worker stop and start again, so +the database dump and the files in the volumes describe the same moment; Postgres +itself keeps running. On a small install that pause is seconds, but count on it +if you run `docsgpt backup` from cron. The archive is written readable only by +the user who took it. + +The archive does **not** include `.env`, because that file holds the install's +secrets. `docsgpt backup --with-settings` puts it in, for when the archive +itself is stored somewhere private. Keep `.env` safe separately otherwise: the +database password in it is what an existing Postgres volume expects. + +Restoring replaces the data in an install: + +```bash +docsgpt restore ~/.docsgpt/server/backups/docsgpt-20260916-120000.tar.gz +``` + +It asks first, then stops the stack, puts the volumes and the database back, and +starts DocsGPT again. `--yes` skips the question for scripts. A backup taken +with a newer DocsGPT is refused, since its data may not fit this version's +schema; upgrade first, or pass `--force` if you know the two match. + +The Postgres data directory itself is not archived: the dump is the database +backup, and copying a directory Postgres is writing to would capture a torn +copy. Caddy's certificates are not archived either, as it obtains them again. + More `docsgpt up` options: `--port`, `--docling` (the image with the docling parser engine and OCR), `--image-tag develop` (follow the `main` branch) and `--adopt` (manage a stack you started from the standalone Compose file in diff --git a/docs/content/changelog.mdx b/docs/content/changelog.mdx index 6e59a1f5..056fb101 100644 --- a/docs/content/changelog.mdx +++ b/docs/content/changelog.mdx @@ -11,6 +11,17 @@ The notable changes in each release. Every release on GitHub also carries [auto-generated notes](https://github.com/arc53/DocsGPT/releases) listing every merged pull request, and [Upgrading](/upgrading) covers the steps an existing deployment has to take. +## Unreleased + +### Back up and restore an install + +`docsgpt backup` writes a dump of the database and a tar of each data volume into one archive, and +`docsgpt restore ` puts them back. The settings file is left out unless +`--with-settings` asks for it, since it holds the install's secrets, and a backup from a newer +DocsGPT is refused unless you pass `--force`. The archive is written readable only by its owner, +and the backend and the worker pause while it is made so the dump and the volume tars match. See +[Backups](/Deploying/Docker-Deploying#backups). + ## 0.21.0 ### Install with one command diff --git a/docsgpt/cli.py b/docsgpt/cli.py index 873cd099..e5a5af2a 100644 --- a/docsgpt/cli.py +++ b/docsgpt/cli.py @@ -230,6 +230,18 @@ def _add_deploy_commands(commands) -> None: uninstall.add_argument("-y", "--yes", action="store_true", help="do not ask for confirmation") uninstall.add_argument("--purge", action="store_true", help="also delete the settings and all data") + backup = stack_command("backup", "backup", "write a backup of the database and the uploaded data") + backup.add_argument("--out", help="directory for the archive (default: /backups)") + backup.add_argument("--with-settings", action="store_true", + help="include .env in the archive; it holds this install's secrets") + + restore = stack_command("restore", "restore", "restore a backup over this install") + restore.add_argument("archive", help="the .tar.gz written by `docsgpt backup`") + restore.add_argument("-y", "--yes", action="store_true", help="do not ask for confirmation") + restore.add_argument("--force", action="store_true", help="restore a backup taken with a newer DocsGPT") + restore.add_argument("--timeout", type=int, default=300, + help="seconds to wait for the API afterwards (default: 300)") + env = stack_command("env", "env", "show, get or set the stack's settings") env_actions = env.add_subparsers(dest="env_action", metavar="") get = env_actions.add_parser("get", help="print one setting") diff --git a/docsgpt/deploy/backup.py b/docsgpt/deploy/backup.py new file mode 100644 index 00000000..e823b9ee --- /dev/null +++ b/docsgpt/deploy/backup.py @@ -0,0 +1,149 @@ +"""The archive ``docsgpt backup`` writes and ``docsgpt restore`` reads. + +One gzipped tar holds a SQL dump of the database, a tar per data volume, and a +manifest saying which version and image the backup came from. The settings file +is left out unless it is asked for: it holds the secrets. + +``postgres_data`` is not tarred, because the dump is the database backup and a +copy of a running data directory would be a torn one. Caddy's volumes are left +out too: they hold certificates it obtains again on the next start. +""" + +from __future__ import annotations + +import io +import json +import os +import tarfile +import time +from collections.abc import Mapping +from datetime import datetime +from pathlib import Path +from typing import Optional + +from docsgpt.deploy.docker import DeployError + +FORMAT = 1 +MANIFEST = "manifest.json" +DUMP = "database.sql" +SETTINGS = "settings.env" +VOLUME_DIR = "volumes" +DATA_VOLUMES = ("indexes", "inputs", "vectors") + + +def archive_name(when: datetime) -> str: + """The file name for a backup taken at ``when``.""" + return f"docsgpt-{when.strftime('%Y%m%d-%H%M%S')}.tar.gz" + + +def volume_member(name: str) -> str: + """Where a volume's tar sits inside the archive.""" + return f"{VOLUME_DIR}/{name}.tar" + + +def write_archive( + path: Path, + *, + dump: Path, + volume_tars: Mapping[str, Path], + manifest: Mapping[str, object], + settings: Optional[Path] = None, +) -> None: + """Write the backup archive; the manifest is added last so a truncated file has none.""" + path.parent.mkdir(parents=True, exist_ok=True) + # 0600 from the start: the dump is the install's data, and --with-settings adds its secrets. + descriptor = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600) + os.fchmod(descriptor, 0o600) + with os.fdopen(descriptor, "wb") as handle, tarfile.open(fileobj=handle, mode="w:gz") as archive: + archive.add(dump, arcname=DUMP) + for name, tar_path in sorted(volume_tars.items()): + archive.add(tar_path, arcname=volume_member(name)) + if settings is not None: + archive.add(settings, arcname=SETTINGS) + body = json.dumps(dict(manifest), indent=2).encode("utf-8") + info = tarfile.TarInfo(MANIFEST) + info.size = len(body) + info.mtime = int(time.time()) + info.mode = 0o600 + archive.addfile(info, io.BytesIO(body)) + + +def read_manifest(path: Path) -> dict: + """The archive's manifest, or a DeployError naming what is wrong with the file.""" + if not path.is_file(): + raise DeployError(f"{path} does not exist") + try: + with tarfile.open(path, "r:gz") as archive: + member = archive.extractfile(MANIFEST) + if member is None: + raise KeyError(MANIFEST) + manifest = json.loads(member.read().decode("utf-8")) + except (tarfile.TarError, KeyError, ValueError, OSError) as exc: + raise DeployError(f"{path} is not a DocsGPT backup: {exc}") from exc + if not isinstance(manifest, dict) or "volumes" not in manifest: + raise DeployError(f"{path} is not a DocsGPT backup: its manifest is missing what to restore") + return manifest + + +def validate(path: Path, manifest: Mapping[str, object]) -> list[str]: + """The volumes to restore, once the archive is known to hold everything it declares. + + Checked before the stack is stopped: a damaged or hand-made archive must fail while DocsGPT is + still running, not after ``docker compose down``. Only the volumes a backup is made of are + accepted, so a manifest cannot name ``postgres_data`` and have it emptied on the way in. + """ + volumes = manifest.get("volumes") + if not isinstance(volumes, list) or not all(isinstance(name, str) for name in volumes): + raise DeployError(f"{path} is not a DocsGPT backup: its manifest does not list the volumes it holds") + unsupported = sorted(set(volumes) - set(DATA_VOLUMES)) + if unsupported: + raise DeployError( + f"{path} names volumes that are not part of a backup: {', '.join(unsupported)}. " + f"A DocsGPT backup holds {', '.join(DATA_VOLUMES)}." + ) + required = [DUMP, *(volume_member(name) for name in volumes)] + try: + with tarfile.open(path, "r:gz") as archive: + present = {member.name for member in archive.getmembers() if member.isfile()} + except (tarfile.TarError, OSError) as exc: + raise DeployError(f"{path} is not a DocsGPT backup: {exc}") from exc + missing = [name for name in required if name not in present] + if missing: + raise DeployError(f"{path} is missing {', '.join(missing)}, so there is nothing to restore from") + return list(volumes) + + +def check_volume_tar(name: str, path: Path) -> None: + """Read a volume tar through, so a damaged one is found before any volume is replaced.""" + try: + with tarfile.open(path, "r:*") as archive: + for _ in archive: + pass + except (tarfile.TarError, OSError) as exc: + raise DeployError(f"the {name} volume in this backup is damaged: {exc}") from exc + + +def extract(path: Path, destination: Path) -> None: + """Unpack the archive into ``destination`` (data filter: no paths outside it, no devices).""" + with tarfile.open(path, "r:gz") as archive: + archive.extractall(destination, filter="data") + + +def _parts(version: str) -> tuple[int, ...]: + numbers = [] + for chunk in str(version).split("."): + digits = "".join(character for character in chunk if character.isdigit()) + numbers.append(int(digits) if digits else 0) + return tuple(numbers) + + +def check_version(manifest: Mapping[str, object], current: str, force: bool) -> None: + """Refuse a backup from a newer DocsGPT: its data may not fit this version's schema.""" + taken_with = str(manifest.get("version") or "") + if force or not taken_with: + return + if _parts(taken_with) > _parts(current): + raise DeployError( + f"this backup is from DocsGPT {taken_with}, newer than the installed {current}. " + "Upgrade first with `docsgpt upgrade`, or pass --force to restore it anyway." + ) diff --git a/docsgpt/deploy/commands.py b/docsgpt/deploy/commands.py index eaa38f44..5aa5fa42 100644 --- a/docsgpt/deploy/commands.py +++ b/docsgpt/deploy/commands.py @@ -9,6 +9,7 @@ import re import shutil import subprocess import sys +import tempfile import webbrowser from collections.abc import Callable, Mapping from dataclasses import dataclass @@ -16,6 +17,7 @@ from datetime import datetime, timezone from pathlib import Path from typing import Any, Optional +from docsgpt.deploy import backup as backup_format from docsgpt.deploy import envfile, stack from docsgpt.deploy.docker import DeployError, Docker, lan_ip, wait_healthy @@ -350,6 +352,149 @@ def env(args, context: Optional[Context] = None) -> int: return 0 +def _stack_image(env: Mapping[str, str]) -> str: + """The image this install runs; the volume tars go through it, so nothing extra is pulled.""" + tag = env.get("DOCSGPT_IMAGE_TAG") or "latest" + return f"arc53/docsgpt:{tag}{env.get('DOCSGPT_IMAGE_VARIANT', '')}" + + +def backup(args, context: Optional[Context] = None) -> int: + """Write a dump of the database and a tar of each data volume into one archive.""" + context = context or Context.default(args) + directory = stack.stack_dir(args.dir) + env = _installed(directory) + if env is None: + return 1 + + out_dir = Path(args.out).expanduser() if args.out else directory / "backups" + taken_at = datetime.now(timezone.utc) + target = out_dir / backup_format.archive_name(taken_at) + image = _stack_image(env) + + try: + print("Pausing the backend and the worker so the database and the files match ...") + context.docker.compose(directory, "stop", "backend", "worker") + _write_backup(args, context, directory, env, target, image, taken_at) + finally: + print("Starting the backend and the worker again ...") + context.docker.compose(directory, "up", "-d", "backend", "worker") + + size = target.stat().st_size / 1_000_000 + print(f"\nBackup written to {target} ({size:.1f} MB)") + if args.with_settings: + print("It contains .env, so it holds this install's secrets: keep it somewhere private.") + else: + print("Settings are not in it; `docsgpt backup --with-settings` includes .env, secrets and all.") + print(f"Restore it with: docsgpt restore {target}") + return 0 + + +def _write_backup(args, context: Context, directory: Path, env, target: Path, image: str, taken_at) -> None: + """Dump the database and the data volumes into ``target``, with the writers stopped.""" + with tempfile.TemporaryDirectory() as workspace: + work = Path(workspace) + dump = work / backup_format.DUMP + print("Dumping the database ...") + with dump.open("w", encoding="utf-8") as handle: + context.docker.compose( + directory, "exec", "-T", "postgres", + "pg_dump", "--clean", "--if-exists", "-U", "docsgpt", "-d", "docsgpt", + stdout=handle, + ) + volume_tars = {} + for name in backup_format.DATA_VOLUMES: + print(f"Archiving the {name} volume ...") + tar_path = work / f"{name}.tar" + context.docker.export_volume(f"{PROJECT}_{name}", tar_path, image) + volume_tars[name] = tar_path + manifest = { + "format": backup_format.FORMAT, + "created_at": taken_at.isoformat(timespec="seconds"), + "version": context.version, + "image_tag": env.get("DOCSGPT_IMAGE_TAG", ""), + "volumes": list(backup_format.DATA_VOLUMES), + "settings_included": bool(args.with_settings), + } + settings = directory / ".env" if args.with_settings else None + backup_format.write_archive(target, dump=dump, volume_tars=volume_tars, manifest=manifest, settings=settings) + + +def _restore_data(context: Context, directory: Path, archive: Path, volumes: list, image: str) -> None: + """Put the archive's volumes and database back, with the stack stopped.""" + with tempfile.TemporaryDirectory() as workspace: + work = Path(workspace) + backup_format.extract(archive, work) + # Read every volume tar through before replacing any of them: a damaged third tar must not + # be discovered with the first two already swapped in. + tars = {} + for name in volumes: + tar_path = work / backup_format.volume_member(name) + if not tar_path.is_file(): + raise DeployError(f"{archive} is missing the {name} volume it says it contains") + backup_format.check_volume_tar(name, tar_path) + tars[name] = tar_path + for name, tar_path in tars.items(): + print(f"Restoring the {name} volume ...") + context.docker.import_volume(f"{PROJECT}_{name}", tar_path, image) + + dump = work / backup_format.DUMP + if not dump.is_file(): + raise DeployError(f"{archive} is missing its database dump") + print("Starting the database ...") + context.docker.compose(directory, "up", "-d", "--wait", "postgres") + print("Restoring the database ...") + with dump.open("r", encoding="utf-8") as handle: + context.docker.compose( + directory, "exec", "-T", "postgres", + "psql", "--quiet", "--set", "ON_ERROR_STOP=on", "-U", "docsgpt", "-d", "docsgpt", + stdin=handle, + ) + + +def restore(args, context: Optional[Context] = None) -> int: + """Put a backup's database and data volumes back over this install.""" + context = context or Context.default(args) + archive = Path(args.archive).expanduser() + manifest = backup_format.read_manifest(archive) + backup_format.check_version(manifest, context.version, args.force) + # Everything the archive declares is checked here, while DocsGPT is still up. + volumes = backup_format.validate(archive, manifest) + + directory = stack.stack_dir(args.dir) + env = _installed(directory) + if env is None: + return 1 + + taken_at = manifest.get("created_at", "an unknown time") + if not args.yes: + if not context.interactive: + raise DeployError("restore needs --yes when there is no terminal to confirm on") + question = f"Replace the data in {directory} with the backup from {taken_at}? This cannot be undone." + if not context.prompter.confirm(question, default=False): + print("Nothing restored.") + return 1 + + image = _stack_image(env) + try: + print("Stopping the stack ...") + context.docker.compose(directory, *_EVERY_PROFILE, "down") + _restore_data(context, directory, archive, volumes, image) + except BaseException: + # The stack is down by now. A corrupt payload inside an otherwise well-formed archive, or a + # statement psql refuses, must not leave DocsGPT stopped: start it again, then report. + print("The restore failed. Starting DocsGPT again ...", file=sys.stderr) + context.docker.compose(directory, "up", "-d", "--remove-orphans", check=False) + raise + + print("Starting DocsGPT ...") + context.docker.compose(directory, "up", "-d", "--remove-orphans") + if not context.wait(stack.health_url(env), args.timeout): + print("DocsGPT did not answer after the restore. See `docsgpt logs backend`.", file=sys.stderr) + return 1 + print(f"\nRestored the backup from {taken_at}. DocsGPT is running at {stack.url(env, context.lan_ip())}") + return 0 + + def _uninstall_hint(installer: str) -> str: return {"uv": "uv tool uninstall docsgpt", "pipx": "pipx uninstall docsgpt"}.get(installer, "pip uninstall docsgpt") diff --git a/docsgpt/deploy/docker.py b/docsgpt/deploy/docker.py index 795aaeb9..cba24375 100644 --- a/docsgpt/deploy/docker.py +++ b/docsgpt/deploy/docker.py @@ -22,10 +22,16 @@ class DeployError(Exception): """A problem the user can act on; the command prints it without a traceback.""" -def run(args: Sequence[str], *, cwd: Optional[Path] = None, capture: bool = False, check: bool = True): - """Run a command, streaming its output unless ``capture``; with ``check`` a failure raises DeployError.""" +def run(args: Sequence[str], *, cwd: Optional[Path] = None, capture: bool = False, check: bool = True, + stdout=None, stdin=None): + """Run a command, streaming its output unless ``capture`` or a file is given for ``stdout``. + + ``stdout`` and ``stdin`` take open files, so a dump goes straight to disk and + back in again without passing through this process. + """ + streams = {"capture_output": True} if capture else {"stdout": stdout, "stdin": stdin} try: - result = subprocess.run(list(args), cwd=cwd, text=True, capture_output=capture, check=False) + result = subprocess.run(list(args), cwd=cwd, text=True, check=False, **streams) except FileNotFoundError as exc: raise DeployError(f"{args[0]} is not installed or not on PATH") from exc if check and result.returncode != 0: @@ -94,9 +100,37 @@ class Docker: raise DeployError("Docker is not running. Start it with `sudo systemctl start docker` and run this again.") raise DeployError("Docker is not running. Start Docker Desktop and run this again.") - def compose(self, directory: Path, *args: str, capture: bool = False, check: bool = True): + def compose(self, directory: Path, *args: str, capture: bool = False, check: bool = True, + stdout=None, stdin=None): """``docker compose `` in ``directory``, which holds the Compose file and its ``.env``.""" - return self._run(["docker", "compose", *args], cwd=directory, capture=capture, check=check) + return self._run(["docker", "compose", *args], cwd=directory, capture=capture, check=check, + stdout=stdout, stdin=stdin) + + def export_volume(self, volume: str, dest: Path, image: str) -> None: + """Write ``volume`` to ``dest`` as a tar, through an image the install already has.""" + dest.parent.mkdir(parents=True, exist_ok=True) + with dest.open("wb") as handle: + self._run( + ["docker", "run", "--rm", "-v", f"{volume}:/data:ro", image, "tar", "cf", "-", "-C", "/data", "."], + stdout=handle, + ) + + def import_volume(self, volume: str, source: Path, image: str) -> None: + """Replace ``volume``'s contents with the tar at ``source``; the volume is created when missing.""" + # Unpack into a throwaway directory inside the container first, so a truncated or corrupt + # tar fails before the live volume is touched rather than halfway through emptying it. The + # container makes the directory itself: the image does not run as root, and a fixed path + # would be both a guess about what is writable and a temp-file smell. + script = ( + 'set -e; stage=$(mktemp -d); tar xf - -C "$stage"; ' + 'find /data -mindepth 1 -delete; tar cf - -C "$stage" . | tar xf - -C /data; ' + 'rm -rf "$stage"' + ) + with source.open("rb") as handle: + self._run( + ["docker", "run", "--rm", "-i", "-v", f"{volume}:/data", image, "sh", "-c", script], + stdin=handle, + ) def volume_exists(self, name: str) -> bool: return self._run(["docker", "volume", "inspect", name], capture=True, check=False).returncode == 0 diff --git a/tests/deploy/test_backup.py b/tests/deploy/test_backup.py new file mode 100644 index 00000000..9c80237a --- /dev/null +++ b/tests/deploy/test_backup.py @@ -0,0 +1,268 @@ +"""`docsgpt backup` and `docsgpt restore`, against a fake Docker.""" + +import io +import json +import stat +import tarfile +from pathlib import Path + +import pytest + +from docsgpt.deploy import backup as backup_module +from docsgpt.deploy.docker import DeployError + +from .test_commands import FakeDocker, FakePrompter, _context, _run + + +def _installed(tmp_path, *extra): + assert _run(["up", "--yes", "--dir", str(tmp_path), *extra], _context()) == 0 + + +def _archive_names(path): + with tarfile.open(path, "r:gz") as archive: + return sorted(member.name for member in archive.getmembers() if member.isfile()) + + +def _manifest(path): + with tarfile.open(path, "r:gz") as archive: + return json.loads(archive.extractfile(backup_module.MANIFEST).read()) + + +def _rewrite(archive, dest, *, volumes=None, drop=(), corrupt=()): + """The same archive with other volumes in its manifest, or with members left out or damaged.""" + with tarfile.open(archive, "r:gz") as source, tarfile.open(dest, "w:gz") as out: + for member in source.getmembers(): + if member.name in drop: + continue + body = source.extractfile(member).read() if member.isfile() else None + if member.name in corrupt: + body = body[:60] + member.size = len(body) + if member.name == backup_module.MANIFEST and volumes is not None: + manifest = json.loads(body.decode()) + manifest["volumes"] = volumes + body = json.dumps(manifest).encode() + member.size = len(body) + out.addfile(member, io.BytesIO(body) if body is not None else None) + return dest + + +def _copy_with_version(archive, version, dest): + """The same archive, with another DocsGPT version written into its manifest.""" + with tarfile.open(archive, "r:gz") as source, tarfile.open(dest, "w:gz") as out: + for member in source.getmembers(): + body = source.extractfile(member).read() if member.isfile() else None + if member.name == backup_module.MANIFEST: + manifest = json.loads(body.decode()) + manifest["version"] = version + body = json.dumps(manifest).encode() + member.size = len(body) + out.addfile(member, io.BytesIO(body) if body is not None else None) + return dest + + +class TestBackup: + def test_writes_a_dump_a_tar_per_volume_and_a_manifest(self, tmp_path, capsys): + _installed(tmp_path) + docker = FakeDocker(volumes={"docsgpt_postgres_data"}) + out = tmp_path / "backups" + assert _run(["backup", "--dir", str(tmp_path), "--out", str(out)], _context(docker)) == 0 + + archives = list(out.glob("docsgpt-*.tar.gz")) + assert len(archives) == 1, archives + names = _archive_names(archives[0]) + assert backup_module.DUMP in names + for volume in ("indexes", "inputs", "vectors"): + assert f"volumes/{volume}.tar" in names + manifest = _manifest(archives[0]) + assert manifest["version"] == "0.21.0" + assert manifest["image_tag"] == "0.21.0" + assert manifest["volumes"] == ["indexes", "inputs", "vectors"] + assert manifest["settings_included"] is False + assert "Backup written to" in capsys.readouterr().out + assert [op for op in docker.volume_ops if op[0] == "export"], docker.volume_ops + + def test_the_settings_file_is_left_out_unless_asked_for(self, tmp_path): + _installed(tmp_path) + out = tmp_path / "backups" + assert _run(["backup", "--dir", str(tmp_path), "--out", str(out)], _context()) == 0 + assert backup_module.SETTINGS not in _archive_names(next(out.glob("*.tar.gz"))) + + with_settings = tmp_path / "with-settings" + argv = ["backup", "--dir", str(tmp_path), "--out", str(with_settings), "--with-settings"] + assert _run(argv, _context()) == 0 + assert backup_module.SETTINGS in _archive_names(next(with_settings.glob("*.tar.gz"))) + + def test_the_database_is_dumped_from_the_running_container(self, tmp_path): + _installed(tmp_path) + docker = FakeDocker() + assert _run(["backup", "--dir", str(tmp_path), "--out", str(tmp_path / "b")], _context(docker)) == 0 + dumps = [args for _, args in docker.calls if "pg_dump" in " ".join(args)] + assert dumps, docker.calls + assert dumps[0][:3] == ["exec", "-T", "postgres"] + + def test_the_archive_is_private_to_whoever_took_it(self, tmp_path): + """It can hold .env, and the dump is the install's data either way.""" + _installed(tmp_path) + out = tmp_path / "backups" + argv = ["backup", "--dir", str(tmp_path), "--out", str(out), "--with-settings"] + assert _run(argv, _context()) == 0 + archive = next(out.glob("*.tar.gz")) + assert stat.S_IMODE(archive.stat().st_mode) == 0o600 + + def test_the_writers_are_stopped_while_the_archive_is_made(self, tmp_path): + """Ingestion writes files and rows; a dump taken alongside live writes would not match them.""" + _installed(tmp_path) + docker = FakeDocker() + assert _run(["backup", "--dir", str(tmp_path), "--out", str(tmp_path / "b")], _context(docker)) == 0 + joined = [" ".join(args) for _, args in docker.calls] + stopped = next(index for index, call in enumerate(joined) if call.startswith("stop backend worker")) + dumped = next(index for index, call in enumerate(joined) if "pg_dump" in call) + started = next(index for index, call in enumerate(joined) if call.startswith("up -d backend worker")) + assert stopped < dumped < started, joined + + def test_the_writers_come_back_even_when_stopping_them_fails(self, tmp_path): + """A stop that fails partway must not leave the backend or the worker down.""" + _installed(tmp_path) + + class FailingStop(FakeDocker): + def compose(self, directory, *args, **kwargs): + if args[:1] == ("stop",): + self.calls.append((Path(directory), list(args))) + raise DeployError("compose stop exploded") + return super().compose(directory, *args, **kwargs) + + docker = FailingStop() + with pytest.raises(DeployError, match="compose stop exploded"): + _run(["backup", "--dir", str(tmp_path), "--out", str(tmp_path / "b")], _context(docker)) + assert any(" ".join(args).startswith("up -d backend worker") for _, args in docker.calls), docker.calls + + def test_the_writers_come_back_even_when_the_dump_fails(self, tmp_path): + _installed(tmp_path) + + class FailingDump(FakeDocker): + def compose(self, directory, *args, **kwargs): + if "pg_dump" in args: + raise DeployError("pg_dump exploded") + return super().compose(directory, *args, **kwargs) + + docker = FailingDump() + with pytest.raises(DeployError, match="pg_dump exploded"): + _run(["backup", "--dir", str(tmp_path), "--out", str(tmp_path / "b")], _context(docker)) + assert any(" ".join(args).startswith("up -d backend worker") for _, args in docker.calls), docker.calls + + def test_without_an_install(self, tmp_path, capsys): + assert _run(["backup", "--dir", str(tmp_path)], _context()) == 1 + assert "docsgpt up" in capsys.readouterr().err + + +class TestRestore: + def _backup(self, tmp_path, *extra): + _installed(tmp_path) + out = tmp_path / "backups" + assert _run(["backup", "--dir", str(tmp_path), "--out", str(out), *extra], _context()) == 0 + return next(out.glob("*.tar.gz")) + + def test_restores_the_volumes_and_the_database(self, tmp_path): + archive = self._backup(tmp_path) + docker = FakeDocker(volumes={"docsgpt_postgres_data"}) + argv = ["restore", str(archive), "--dir", str(tmp_path), "--yes"] + assert _run(argv, _context(docker)) == 0 + joined = [" ".join(args) for _, args in docker.calls] + assert any(call.startswith("--profile https down") for call in joined), joined + assert any("psql" in call for call in joined), joined + assert any(call.startswith("up -d") for call in joined), joined + + def test_asks_before_replacing_data(self, tmp_path): + archive = self._backup(tmp_path) + docker = FakeDocker(volumes={"docsgpt_postgres_data"}) + prompter = FakePrompter([False]) + assert _run(["restore", str(archive), "--dir", str(tmp_path)], _context(docker, prompter, interactive=True)) == 1 + assert docker.calls == [] + + def test_refuses_an_archive_that_is_not_a_docsgpt_backup(self, tmp_path): + stray = tmp_path / "stray.tar.gz" + with tarfile.open(stray, "w:gz") as archive: + note = tmp_path / "note.txt" + note.write_text("not a backup") + archive.add(note, arcname="note.txt") + with pytest.raises(DeployError, match="not a DocsGPT backup"): + _run(["restore", str(stray), "--dir", str(tmp_path), "--yes"], _context()) + + def test_a_missing_archive(self, tmp_path): + with pytest.raises(DeployError, match="does not exist"): + _run(["restore", str(tmp_path / "nope.tar.gz"), "--dir", str(tmp_path), "--yes"], _context()) + + def test_a_manifest_naming_another_volume_is_refused(self, tmp_path): + """import_volume empties what it is given, so a hand-made manifest must not name postgres_data.""" + crafted = _rewrite(self._backup(tmp_path), tmp_path / "crafted.tar.gz", volumes=["postgres_data"]) + docker = FakeDocker(volumes={"docsgpt_postgres_data"}) + with pytest.raises(DeployError, match="not part of a backup"): + _run(["restore", str(crafted), "--dir", str(tmp_path), "--yes"], _context(docker)) + assert docker.calls == [], "nothing was stopped" + assert docker.volume_ops == [], "and nothing was touched" + + def test_a_damaged_archive_fails_before_the_stack_is_stopped(self, tmp_path): + """Discovering a missing member after `compose down` would leave DocsGPT down for nothing.""" + without_dump = _rewrite(self._backup(tmp_path), tmp_path / "nodump.tar.gz", drop=(backup_module.DUMP,)) + docker = FakeDocker(volumes={"docsgpt_postgres_data"}) + with pytest.raises(DeployError, match="nothing to restore"): + _run(["restore", str(without_dump), "--dir", str(tmp_path), "--yes"], _context(docker)) + assert docker.calls == [], "DocsGPT is still running" + + def test_psql_stops_at_the_first_failing_statement(self, tmp_path): + """Without it psql runs on after an error and a half-restored database looks like success.""" + archive = self._backup(tmp_path) + docker = FakeDocker(volumes={"docsgpt_postgres_data"}) + assert _run(["restore", str(archive), "--dir", str(tmp_path), "--yes"], _context(docker)) == 0 + psql = next(args for _, args in docker.calls if "psql" in args) + assert "ON_ERROR_STOP=on" in psql + + def test_a_failure_after_the_stack_is_down_starts_it_again(self, tmp_path, capsys): + """A corrupt payload only shows up once the stack is down; it must not be left stopped.""" + archive = self._backup(tmp_path) + + class FailingImport(FakeDocker): + def import_volume(self, volume, source, image): + raise DeployError("tar: unexpected EOF in archive") + + docker = FailingImport(volumes={"docsgpt_postgres_data"}) + with pytest.raises(DeployError, match="unexpected EOF"): + _run(["restore", str(archive), "--dir", str(tmp_path), "--yes"], _context(docker)) + joined = [" ".join(args) for _, args in docker.calls] + assert any(call.startswith("up -d --remove-orphans") for call in joined), joined + assert "Starting DocsGPT again" in capsys.readouterr().err + + def test_a_shutdown_that_fails_still_starts_the_stack_again(self, tmp_path): + """`compose down` can fail with containers already stopped; the stack must not stay down.""" + archive = self._backup(tmp_path) + + class FailingDown(FakeDocker): + def compose(self, directory, *args, **kwargs): + if "down" in args: + self.calls.append((Path(directory), list(args))) + raise DeployError("compose down exploded") + return super().compose(directory, *args, **kwargs) + + docker = FailingDown(volumes={"docsgpt_postgres_data"}) + with pytest.raises(DeployError, match="compose down exploded"): + _run(["restore", str(archive), "--dir", str(tmp_path), "--yes"], _context(docker)) + joined = [" ".join(args) for _, args in docker.calls] + assert any(call.startswith("up -d --remove-orphans") for call in joined), joined + + def test_a_damaged_volume_tar_is_found_before_any_volume_is_replaced(self, tmp_path): + """vectors sorts last, so a per-volume check would already have swapped indexes and inputs.""" + archive = _rewrite(self._backup(tmp_path), tmp_path / "bad-volume.tar.gz", + corrupt=(backup_module.volume_member("vectors"),)) + docker = FakeDocker(volumes={"docsgpt_postgres_data"}) + with pytest.raises(DeployError, match="damaged"): + _run(["restore", str(archive), "--dir", str(tmp_path), "--yes"], _context(docker)) + assert [op for op in docker.volume_ops if op[0] == "import"] == [], docker.volume_ops + + def test_a_newer_backup_is_refused_without_force(self, tmp_path): + """Restoring a 0.22 backup into 0.21 would hand an older schema newer data.""" + newer = _copy_with_version(self._backup(tmp_path), "0.22.0", tmp_path / "newer.tar.gz") + with pytest.raises(DeployError, match="newer"): + _run(["restore", str(newer), "--dir", str(tmp_path), "--yes"], _context(FakeDocker())) + argv = ["restore", str(newer), "--dir", str(tmp_path), "--yes", "--force"] + assert _run(argv, _context(FakeDocker())) == 0 diff --git a/tests/deploy/test_commands.py b/tests/deploy/test_commands.py index 30faa350..b73fedb2 100644 --- a/tests/deploy/test_commands.py +++ b/tests/deploy/test_commands.py @@ -1,8 +1,10 @@ """`docsgpt up` and the commands that manage the stack, against a fake Docker.""" +import io import json import subprocess import sys +import tarfile from pathlib import Path import pytest @@ -19,17 +21,38 @@ class FakeDocker: self.volumes = set(volumes) self.dirs = {Path(d) for d in project_dirs} self.calls = [] + self.volume_ops = [] self.preflights = 0 def preflight(self, interactive=False): self.preflights += 1 - def compose(self, directory, *args, capture=False, check=True): + def compose(self, directory, *args, capture=False, check=True, stdout=None, stdin=None): self.calls.append((Path(directory), list(args))) if "down" in args and "-v" in args: self.volumes.clear() + if stdout is not None: + stdout.write("-- fake pg_dump\n") + if stdin is not None: + self.restored_sql = stdin.read() return subprocess.CompletedProcess(["docker", "compose", *args], 0, stdout="", stderr="") + def export_volume(self, volume, dest, image): + """Write a small tar, as the real one does with `docker run ... tar cf -`.""" + self.volume_ops.append(("export", volume, image)) + dest = Path(dest) + dest.parent.mkdir(parents=True, exist_ok=True) + with tarfile.open(dest, "w") as tar: + body = f"{volume} contents".encode() + info = tarfile.TarInfo(f"{volume}.marker") + info.size = len(body) + tar.addfile(info, io.BytesIO(body)) + + def import_volume(self, volume, source, image): + self.volume_ops.append(("import", volume, image)) + self.volumes.add(volume) + assert Path(source).is_file(), source + def volume_exists(self, name): return name in self.volumes diff --git a/tests/deploy/test_docker.py b/tests/deploy/test_docker.py index 8e42556e..f42d57c9 100644 --- a/tests/deploy/test_docker.py +++ b/tests/deploy/test_docker.py @@ -15,9 +15,11 @@ class FakeRunner: def __init__(self, answers=None): self.answers = list((answers or {}).items()) self.calls = [] + self.streams = [] - def __call__(self, args, *, cwd=None, capture=False, check=True): + def __call__(self, args, *, cwd=None, capture=False, check=True, stdout=None, stdin=None): self.calls.append((list(args), cwd)) + self.streams.append((stdout, stdin)) for prefix, answer in self.answers: if list(args[: len(prefix)]) == list(prefix): if callable(answer): @@ -105,6 +107,36 @@ class TestQueries: assert "label=com.docker.compose.project=docsgpt" in args +class TestVolumes: + def test_export_writes_the_volume_through_the_stack_image(self, tmp_path): + runner = FakeRunner() + dest = tmp_path / "volumes" / "inputs.tar" + _docker(runner).export_volume("docsgpt_inputs", dest, "arc53/docsgpt:0.21.0") + args, _ = runner.calls[0] + assert args[:4] == ["docker", "run", "--rm", "-v"] + assert args[4] == "docsgpt_inputs:/data:ro" + assert args[5] == "arc53/docsgpt:0.21.0" + assert args[6:] == ["tar", "cf", "-", "-C", "/data", "."] + assert dest.is_file(), "the tar is written to the destination" + assert runner.streams[0][0] is not None, "stdout goes to the file, not through this process" + + def test_import_replaces_the_volume_contents(self, tmp_path): + source = tmp_path / "inputs.tar" + source.write_bytes(b"tar") + runner = FakeRunner() + _docker(runner).import_volume("docsgpt_inputs", source, "arc53/docsgpt:0.21.0") + args, _ = runner.calls[0] + assert "docsgpt_inputs:/data" in args + assert args[-2] == "-c" + assert "tar xf - -C /data" in args[-1] + command = args[-1] + assert "mktemp -d" in command, "the container picks the staging directory, not a fixed path" + assert command.index('tar xf - -C "$stage"') < command.index("find /data -mindepth 1 -delete"), ( + "the incoming tar is unpacked outside the volume first, so a corrupt one leaves it alone" + ) + assert runner.streams[0][1] is not None, "the tar is fed in on stdin" + + class TestWaitHealthy: def test_succeeds_once_the_api_answers(self): attempts = {"n": 0}