fix: start DocsGPT again when a restore fails after the stack is down

Validating the archive catches a damaged one while DocsGPT is still up, but a
well-formed archive can still hold a corrupt volume tar or a dump statement
psql refuses, and those only surface once the stack is down. The work after
the shutdown now runs inside an error boundary that starts the stack again
before the failure is reported, so a failed restore never leaves the install
stopped.
This commit is contained in:
Alex committed 2026-09-16 22:03:48 +01:00
1 parent 4347636496
commit e269743bf8
2 files changed
+49 -22

No files matched your search

+34 -22
View File
@@ -419,6 +419,32 @@ def _write_backup(args, context: Context, directory: Path, env, target: Path, im
backup_format.write_archive(target, dump=dump, volume_tars=volume_tars, manifest=manifest, settings=settings)
def _restore_data(context: Context, directory: Path, archive: Path, volumes: list, image: str) -> None:
"""Put the archive's volumes and database back, with the stack stopped."""
with tempfile.TemporaryDirectory() as workspace:
work = Path(workspace)
backup_format.extract(archive, work)
for name in volumes:
tar_path = work / backup_format.volume_member(name)
if not tar_path.is_file():
raise DeployError(f"{archive} is missing the {name} volume it says it contains")
print(f"Restoring the {name} volume ...")
context.docker.import_volume(f"{PROJECT}_{name}", tar_path, image)
dump = work / backup_format.DUMP
if not dump.is_file():
raise DeployError(f"{archive} is missing its database dump")
print("Starting the database ...")
context.docker.compose(directory, "up", "-d", "--wait", "postgres")
print("Restoring the database ...")
with dump.open("r", encoding="utf-8") as handle:
context.docker.compose(
directory, "exec", "-T", "postgres",
"psql", "--quiet", "--set", "ON_ERROR_STOP=on", "-U", "docsgpt", "-d", "docsgpt",
stdin=handle,
)
def restore(args, context: Optional[Context] = None) -> int:
"""Put a backup's database and data volumes back over this install."""
context = context or Context.default(args)
@@ -446,28 +472,14 @@ def restore(args, context: Optional[Context] = None) -> int:
print("Stopping the stack ...")
context.docker.compose(directory, *_EVERY_PROFILE, "down")
with tempfile.TemporaryDirectory() as workspace:
work = Path(workspace)
backup_format.extract(archive, work)
for name in volumes:
tar_path = work / backup_format.volume_member(name)
if not tar_path.is_file():
raise DeployError(f"{archive} is missing the {name} volume it says it contains")
print(f"Restoring the {name} volume ...")
context.docker.import_volume(f"{PROJECT}_{name}", tar_path, image)
dump = work / backup_format.DUMP
if not dump.is_file():
raise DeployError(f"{archive} is missing its database dump")
print("Starting the database ...")
context.docker.compose(directory, "up", "-d", "--wait", "postgres")
print("Restoring the database ...")
with dump.open("r", encoding="utf-8") as handle:
context.docker.compose(
directory, "exec", "-T", "postgres",
"psql", "--quiet", "--set", "ON_ERROR_STOP=on", "-U", "docsgpt", "-d", "docsgpt",
stdin=handle,
)
try:
_restore_data(context, directory, archive, volumes, image)
except BaseException:
# The stack is down by now. A corrupt payload inside an otherwise well-formed archive, or a
# statement psql refuses, must not leave DocsGPT stopped: start it again, then report.
print("The restore failed. Starting DocsGPT again ...", file=sys.stderr)
context.docker.compose(directory, "up", "-d", "--remove-orphans", check=False)
raise
print("Starting DocsGPT ...")
context.docker.compose(directory, "up", "-d", "--remove-orphans")
+15
View File
@@ -198,6 +198,21 @@ class TestRestore:
psql = next(args for _, args in docker.calls if "psql" in args)
assert "ON_ERROR_STOP=on" in psql
def test_a_failure_after_the_stack_is_down_starts_it_again(self, tmp_path, capsys):
"""A corrupt payload only shows up once the stack is down; it must not be left stopped."""
archive = self._backup(tmp_path)
class FailingImport(FakeDocker):
def import_volume(self, volume, source, image):
raise DeployError("tar: unexpected EOF in archive")
docker = FailingImport(volumes={"docsgpt_postgres_data"})
with pytest.raises(DeployError, match="unexpected EOF"):
_run(["restore", str(archive), "--dir", str(tmp_path), "--yes"], _context(docker))
joined = [" ".join(args) for _, args in docker.calls]
assert any(call.startswith("up -d --remove-orphans") for call in joined), joined
assert "Starting DocsGPT again" in capsys.readouterr().err
def test_a_newer_backup_is_refused_without_force(self, tmp_path):
"""Restoring a 0.22 backup into 0.21 would hand an older schema newer data."""
newer = _copy_with_version(self._backup(tmp_path), "0.22.0", tmp_path / "newer.tar.gz")