fix(backups): stop containers during volume backup creation

Track and recover stopped containers across interrupted backup executions.

app/Jobs/ScheduledJobManager.php

@@ -112,7 +112,7 @@ class ScheduledJobManager implements ShouldQueue
         }
 
         try {
-            $this->recoverPausedVolumeBackupContainers();
+            $this->recoverStoppedVolumeBackupContainers();
             $this->processScheduledVolumeBackups();
         } catch (\Exception $e) {
             Log::channel('scheduled-errors')->error('Failed to process scheduled volume backups', [
@@ -365,11 +365,11 @@ class ScheduledJobManager implements ShouldQueue
             });
     }
 
-    private function recoverPausedVolumeBackupContainers(): void
+    private function recoverStoppedVolumeBackupContainers(): void
     {
         ScheduledVolumeBackupExecution::query()
             ->where(fn (Builder $query) => $query
-                ->where('pause_recovery_pending', true)
+                ->where('stop_recovery_pending', true)
                 ->orWhere('s3_cleanup_pending', true))
             ->chunkById(self::CHUNK_SIZE, function ($executions): void {
                 foreach ($executions as $execution) {
@@ -385,7 +385,7 @@ class ScheduledJobManager implements ShouldQueue
 
             if ($backup->executions()
                 ->where(fn (Builder $query) => $query
-                    ->where('pause_recovery_pending', true)
+                    ->where('stop_recovery_pending', true)
                     ->orWhere('s3_cleanup_pending', true))
                 ->exists()) {
                 $this->skippedCount++;

app/Jobs/VolumeBackupJob.php

@@ -83,11 +83,11 @@ class VolumeBackupJob implements ShouldBeEncrypted, ShouldQueue
                 .' '.escapeshellarg($image)
                 .' tar -czf - -C /volume . > '.escapeshellarg($backupLocation);
 
-            if ($this->backup->pause_during_backup) {
+            if ($this->backup->stop_during_backup) {
                 $containers = $this->containersUsingVolume($source, $server);
                 if ($containers !== []) {
-                    $this->execution->update(['pause_recovery_pending' => true]);
-                    $archiveCommand = $this->archiveWithPausedContainers(
+                    $this->execution->update(['stop_recovery_pending' => true]);
+                    $archiveCommand = $this->archiveWithStoppedContainers(
                         $archiveCommand,
                         $containers,
                         VolumeBackupRecoveryJob::stateFile($this->execution),
@@ -101,8 +101,8 @@ class VolumeBackupJob implements ShouldBeEncrypted, ShouldQueue
                 $archiveCommand,
             ], $server, timeout: $this->timeout, disableMultiplexing: true);
             $this->execution->update([
-                'pause_container_ids' => null,
-                'pause_recovery_pending' => false,
+                'stop_container_ids' => null,
+                'stop_recovery_pending' => false,
             ]);
 
             $size = (int) instant_remote_process(
@@ -253,7 +253,7 @@ class VolumeBackupJob implements ShouldBeEncrypted, ShouldQueue
     /**
      * @param  array<int, string>  $containers
      */
-    private function archiveWithPausedContainers(string $archiveCommand, array $containers, string $stateFile): string
+    private function archiveWithStoppedContainers(string $archiveCommand, array $containers, string $stateFile): string
     {
         if ($containers === []) {
             return $archiveCommand;
@@ -263,9 +263,12 @@ class VolumeBackupJob implements ShouldBeEncrypted, ShouldQueue
         $script = "set -eu\n"
             .'state_file='.escapeshellarg($stateFile)."\n"
             .": > \"\$state_file\"\n"
-            ."cleanup() { status=\$?; trap - EXIT HUP INT TERM; while IFS= read -r container; do docker unpause \"\$container\" >/dev/null || status=1; done < \"\$state_file\"; [ \$status -eq 0 ] && rm -f \"\$state_file\"; exit \$status; }\n"
-            ."trap cleanup EXIT HUP INT TERM\n"
-            ."for container in {$containerList}; do echo \"\$container\" >> \"\$state_file\"; docker pause \"\$container\" >/dev/null; done\n"
+            ."cleanup() { status=\$?; trap - EXIT HUP INT TERM; while IFS= read -r container; do docker start \"\$container\" >/dev/null || status=1; done < \"\$state_file\"; [ \$status -eq 0 ] && rm -f \"\$state_file\"; exit \$status; }\n"
+            ."trap cleanup EXIT\n"
+            ."trap 'exit 129' HUP\n"
+            ."trap 'exit 130' INT\n"
+            ."trap 'exit 143' TERM\n"
+            ."for container in {$containerList}; do echo \"\$container\" >> \"\$state_file\"; docker stop \"\$container\" >/dev/null; done\n"
             .$archiveCommand;
 
         return 'sh -c '.escapeshellarg($script);
@@ -273,7 +276,7 @@ class VolumeBackupJob implements ShouldBeEncrypted, ShouldQueue
 
     private function recoverIncompleteBackup(ScheduledVolumeBackupExecution $execution): string
     {
-        if (! $execution->pause_recovery_pending && ! $execution->s3_cleanup_pending) {
+        if (! $execution->stop_recovery_pending && ! $execution->s3_cleanup_pending) {
             return '';
         }
 

app/Jobs/VolumeBackupRecoveryJob.php

@@ -48,7 +48,7 @@ class VolumeBackupRecoveryJob implements ShouldBeEncrypted, ShouldQueue
     {
         $execution->loadMissing('scheduledVolumeBackup.volume.resource');
 
-        if ($execution->pause_recovery_pending) {
+        if ($execution->stop_recovery_pending) {
             self::recoverContainers($execution);
         }
 
@@ -75,22 +75,22 @@ class VolumeBackupRecoveryJob implements ShouldBeEncrypted, ShouldQueue
             ->filter(fn (string $container): bool => preg_match('/^[a-f0-9]{6,64}$/i', $container) === 1)
             ->values()
             ->all();
-        $execution->update(['pause_container_ids' => $containers]);
+        $execution->update(['stop_container_ids' => $containers]);
 
         $remainingFile = $stateFile.'.remaining';
         $script = 'status=0; : > '.escapeshellarg($remainingFile).'; '
             .'if [ -f '.escapeshellarg($stateFile).' ]; then while IFS= read -r container; do '
-            .'[ -z "$container" ] && continue; paused=$(docker inspect --format \'{{.State.Paused}}\' "$container" 2>/dev/null) '
+            .'[ -z "$container" ] && continue; running=$(docker inspect --format \'{{.State.Running}}\' "$container" 2>/dev/null) '
             .'|| { echo "$container" >> '.escapeshellarg($remainingFile).'; status=1; continue; }; '
-            .'if [ "$paused" = true ] && ! docker unpause "$container" >/dev/null; then echo "$container" >> '
+            .'if [ "$running" != true ] && ! docker start "$container" >/dev/null; then echo "$container" >> '
             .escapeshellarg($remainingFile).'; status=1; fi; done < '.escapeshellarg($stateFile).'; fi; '
             .'if [ -s '.escapeshellarg($remainingFile).' ]; then mv '.escapeshellarg($remainingFile).' '.escapeshellarg($stateFile)
             .'; else rm -f '.escapeshellarg($stateFile).' '.escapeshellarg($remainingFile).'; fi; exit $status';
 
         instant_remote_process(['sh -c '.escapeshellarg($script)], $server, disableMultiplexing: true);
         $execution->update([
-            'pause_container_ids' => null,
-            'pause_recovery_pending' => false,
+            'stop_container_ids' => null,
+            'stop_recovery_pending' => false,
         ]);
     }
 
@@ -112,6 +112,6 @@ class VolumeBackupRecoveryJob implements ShouldBeEncrypted, ShouldQueue
 
     public static function stateFile(ScheduledVolumeBackupExecution $execution): string
     {
-        return '/tmp/coolify-volume-backup-'.$execution->uuid.'.paused';
+        return '/tmp/coolify-volume-backup-'.$execution->uuid.'.stopped';
     }
 }