fix: cover the shutdown calls and check every volume before replacing one

Both shutdown calls sat outside the recovery that undoes them: a `compose down`
that failed partway left the stack down, and a `compose stop` that failed left
the backend and worker stopped. Each now runs inside its own try.

A restore also replaced volumes one at a time, checking each tar as it reached
it, so a damaged third payload was found with the first two already swapped in.
Every declared tar is read through first, and the imports start only once they
all come out whole.
This commit is contained in:
Alex committed 2026-09-16 22:20:40 +01:00
1 parent 8cfa3fbd18
commit f90c442a41
3 files changed
+68 -7

No files matched your search

+10
View File
@@ -113,6 +113,16 @@ def validate(path: Path, manifest: Mapping[str, object]) -> list[str]:
return list(volumes)
def check_volume_tar(name: str, path: Path) -> None:
"""Read a volume tar through, so a damaged one is found before any volume is replaced."""
try:
with tarfile.open(path, "r:*") as archive:
for _ in archive:
pass
except (tarfile.TarError, OSError) as exc:
raise DeployError(f"the {name} volume in this backup is damaged: {exc}") from exc
def extract(path: Path, destination: Path) -> None:
"""Unpack the archive into ``destination`` (data filter: no paths outside it, no devices)."""
with tarfile.open(path, "r:gz") as archive:
+10 -5
View File
@@ -371,9 +371,9 @@ def backup(args, context: Optional[Context] = None) -> int:
target = out_dir / backup_format.archive_name(taken_at)
image = _stack_image(env)
print("Pausing the backend and the worker so the database and the files match ...")
context.docker.compose(directory, "stop", "backend", "worker")
try:
print("Pausing the backend and the worker so the database and the files match ...")
context.docker.compose(directory, "stop", "backend", "worker")
_write_backup(args, context, directory, env, target, image, taken_at)
finally:
print("Starting the backend and the worker again ...")
@@ -424,10 +424,16 @@ def _restore_data(context: Context, directory: Path, archive: Path, volumes: lis
with tempfile.TemporaryDirectory() as workspace:
work = Path(workspace)
backup_format.extract(archive, work)
# Read every volume tar through before replacing any of them: a damaged third tar must not
# be discovered with the first two already swapped in.
tars = {}
for name in volumes:
tar_path = work / backup_format.volume_member(name)
if not tar_path.is_file():
raise DeployError(f"{archive} is missing the {name} volume it says it contains")
backup_format.check_volume_tar(name, tar_path)
tars[name] = tar_path
for name, tar_path in tars.items():
print(f"Restoring the {name} volume ...")
context.docker.import_volume(f"{PROJECT}_{name}", tar_path, image)
@@ -469,10 +475,9 @@ def restore(args, context: Optional[Context] = None) -> int:
return 1
image = _stack_image(env)
print("Stopping the stack ...")
context.docker.compose(directory, *_EVERY_PROFILE, "down")
try:
print("Stopping the stack ...")
context.docker.compose(directory, *_EVERY_PROFILE, "down")
_restore_data(context, directory, archive, volumes, image)
except BaseException:
# The stack is down by now. A corrupt payload inside an otherwise well-formed archive, or a