Merge feat/install-4-backup into feat/install-5-native

docsgpt/deploy/backup.py

@@ -13,6 +13,7 @@ from __future__ import annotations
 
 import io
 import json
+import os
 import tarfile
 import time
 from collections.abc import Mapping
@@ -50,7 +51,10 @@ def write_archive(
 ) -> None:
     """Write the backup archive; the manifest is added last so a truncated file has none."""
     path.parent.mkdir(parents=True, exist_ok=True)
-    with tarfile.open(path, "w:gz") as archive:
+    # 0600 from the start: the dump is the install's data, and --with-settings adds its secrets.
+    descriptor = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600)
+    os.fchmod(descriptor, 0o600)
+    with os.fdopen(descriptor, "wb") as handle, tarfile.open(fileobj=handle, mode="w:gz") as archive:
         archive.add(dump, arcname=DUMP)
         for name, tar_path in sorted(volume_tars.items()):
             archive.add(tar_path, arcname=volume_member(name))
@@ -81,6 +85,34 @@ def read_manifest(path: Path) -> dict:
     return manifest
 
 
+def validate(path: Path, manifest: Mapping[str, object]) -> list[str]:
+    """The volumes to restore, once the archive is known to hold everything it declares.
+
+    Checked before the stack is stopped: a damaged or hand-made archive must fail while DocsGPT is
+    still running, not after ``docker compose down``. Only the volumes a backup is made of are
+    accepted, so a manifest cannot name ``postgres_data`` and have it emptied on the way in.
+    """
+    volumes = manifest.get("volumes")
+    if not isinstance(volumes, list) or not all(isinstance(name, str) for name in volumes):
+        raise DeployError(f"{path} is not a DocsGPT backup: its manifest does not list the volumes it holds")
+    unsupported = sorted(set(volumes) - set(DATA_VOLUMES))
+    if unsupported:
+        raise DeployError(
+            f"{path} names volumes that are not part of a backup: {', '.join(unsupported)}. "
+            f"A DocsGPT backup holds {', '.join(DATA_VOLUMES)}."
+        )
+    required = [DUMP, *(volume_member(name) for name in volumes)]
+    try:
+        with tarfile.open(path, "r:gz") as archive:
+            present = {member.name for member in archive.getmembers() if member.isfile()}
+    except (tarfile.TarError, OSError) as exc:
+        raise DeployError(f"{path} is not a DocsGPT backup: {exc}") from exc
+    missing = [name for name in required if name not in present]
+    if missing:
+        raise DeployError(f"{path} is missing {', '.join(missing)}, so there is nothing to restore from")
+    return list(volumes)
+
+
 def extract(path: Path, destination: Path) -> None:
     """Unpack the archive into ``destination`` (data filter: no paths outside it, no devices)."""
     with tarfile.open(path, "r:gz") as archive:

docsgpt/deploy/commands.py

@@ -527,6 +527,26 @@ def backup(args, context: Optional[Context] = None) -> int:
     target = out_dir / backup_format.archive_name(taken_at)
     image = _stack_image(env)
 
+    print("Pausing the backend and the worker so the database and the files match ...")
+    context.docker.compose(directory, "stop", "backend", "worker")
+    try:
+        _write_backup(args, context, directory, env, target, image, taken_at)
+    finally:
+        print("Starting the backend and the worker again ...")
+        context.docker.compose(directory, "up", "-d", "backend", "worker")
+
+    size = target.stat().st_size / 1_000_000
+    print(f"\nBackup written to {target} ({size:.1f} MB)")
+    if args.with_settings:
+        print("It contains .env, so it holds this install's secrets: keep it somewhere private.")
+    else:
+        print("Settings are not in it; `docsgpt backup --with-settings` includes .env, secrets and all.")
+    print(f"Restore it with: docsgpt restore {target}")
+    return 0
+
+
+def _write_backup(args, context: Context, directory: Path, env, target: Path, image: str, taken_at) -> None:
+    """Dump the database and the data volumes into ``target``, with the writers stopped."""
     with tempfile.TemporaryDirectory() as workspace:
         work = Path(workspace)
         dump = work / backup_format.DUMP
@@ -554,15 +574,6 @@ def backup(args, context: Optional[Context] = None) -> int:
         settings = directory / ".env" if args.with_settings else None
         backup_format.write_archive(target, dump=dump, volume_tars=volume_tars, manifest=manifest, settings=settings)
 
-    size = target.stat().st_size / 1_000_000
-    print(f"\nBackup written to {target} ({size:.1f} MB)")
-    if args.with_settings:
-        print("It contains .env, so it holds this install's secrets: keep it somewhere private.")
-    else:
-        print("Settings are not in it; `docsgpt backup --with-settings` includes .env, secrets and all.")
-    print(f"Restore it with: docsgpt restore {target}")
-    return 0
-
 
 def restore(args, context: Optional[Context] = None) -> int:
     """Put a backup's database and data volumes back over this install."""
@@ -570,6 +581,8 @@ def restore(args, context: Optional[Context] = None) -> int:
     archive = Path(args.archive).expanduser()
     manifest = backup_format.read_manifest(archive)
     backup_format.check_version(manifest, context.version, args.force)
+    # Everything the archive declares is checked here, while DocsGPT is still up.
+    volumes = backup_format.validate(archive, manifest)
 
     directory = stack.stack_dir(args.dir)
     env = _installed(directory)
@@ -592,7 +605,7 @@ def restore(args, context: Optional[Context] = None) -> int:
     with tempfile.TemporaryDirectory() as workspace:
         work = Path(workspace)
         backup_format.extract(archive, work)
-        for name in manifest.get("volumes", []):
+        for name in volumes:
             tar_path = work / backup_format.volume_member(name)
             if not tar_path.is_file():
                 raise DeployError(f"{archive} is missing the {name} volume it says it contains")
@@ -608,7 +621,7 @@ def restore(args, context: Optional[Context] = None) -> int:
         with dump.open("r", encoding="utf-8") as handle:
             context.docker.compose(
                 directory, "exec", "-T", "postgres",
-                "psql", "--quiet", "-U", "docsgpt", "-d", "docsgpt",
+                "psql", "--quiet", "--set", "ON_ERROR_STOP=on", "-U", "docsgpt", "-d", "docsgpt",
                 stdin=handle,
             )
 

docsgpt/deploy/docker.py

@@ -119,8 +119,11 @@ class Docker:
         """Replace ``volume``'s contents with the tar at ``source``; the volume is created when missing."""
         with source.open("rb") as handle:
             self._run(
-                ["docker", "run", "--rm", "-i", "-v", f"{volume}:/data", image,
-                 "sh", "-c", "find /data -mindepth 1 -delete && tar xf - -C /data"],
+                ["docker", "run", "--rm", "-i", "-v", f"{volume}:/data", image, "sh", "-c",
+                 # Unpack into the container's own filesystem first: a truncated or corrupt tar must
+                 # fail before the live volume is touched, not halfway through emptying it.
+                 "set -e; rm -rf /stage; mkdir /stage; tar xf - -C /stage; "
+                 "find /data -mindepth 1 -delete; tar cf - -C /stage . | tar xf - -C /data"],
                 stdin=handle,
             )