fix(backups): stop containers during volume backup creation

Track and recover stopped containers across interrupted backup executions.

app/Jobs/ScheduledJobManager.php

@@ -112,7 +112,7 @@ class ScheduledJobManager implements ShouldQueue
         }
 
         try {
-            $this->recoverPausedVolumeBackupContainers();
+            $this->recoverStoppedVolumeBackupContainers();
             $this->processScheduledVolumeBackups();
         } catch (\Exception $e) {
             Log::channel('scheduled-errors')->error('Failed to process scheduled volume backups', [
@@ -365,11 +365,11 @@ class ScheduledJobManager implements ShouldQueue
             });
     }
 
-    private function recoverPausedVolumeBackupContainers(): void
+    private function recoverStoppedVolumeBackupContainers(): void
     {
         ScheduledVolumeBackupExecution::query()
             ->where(fn (Builder $query) => $query
-                ->where('pause_recovery_pending', true)
+                ->where('stop_recovery_pending', true)
                 ->orWhere('s3_cleanup_pending', true))
             ->chunkById(self::CHUNK_SIZE, function ($executions): void {
                 foreach ($executions as $execution) {
@@ -385,7 +385,7 @@ class ScheduledJobManager implements ShouldQueue
 
             if ($backup->executions()
                 ->where(fn (Builder $query) => $query
-                    ->where('pause_recovery_pending', true)
+                    ->where('stop_recovery_pending', true)
                     ->orWhere('s3_cleanup_pending', true))
                 ->exists()) {
                 $this->skippedCount++;

app/Jobs/VolumeBackupJob.php

@@ -83,11 +83,11 @@ class VolumeBackupJob implements ShouldBeEncrypted, ShouldQueue
                 .' '.escapeshellarg($image)
                 .' tar -czf - -C /volume . > '.escapeshellarg($backupLocation);
 
-            if ($this->backup->pause_during_backup) {
+            if ($this->backup->stop_during_backup) {
                 $containers = $this->containersUsingVolume($source, $server);
                 if ($containers !== []) {
-                    $this->execution->update(['pause_recovery_pending' => true]);
-                    $archiveCommand = $this->archiveWithPausedContainers(
+                    $this->execution->update(['stop_recovery_pending' => true]);
+                    $archiveCommand = $this->archiveWithStoppedContainers(
                         $archiveCommand,
                         $containers,
                         VolumeBackupRecoveryJob::stateFile($this->execution),
@@ -101,8 +101,8 @@ class VolumeBackupJob implements ShouldBeEncrypted, ShouldQueue
                 $archiveCommand,
             ], $server, timeout: $this->timeout, disableMultiplexing: true);
             $this->execution->update([
-                'pause_container_ids' => null,
-                'pause_recovery_pending' => false,
+                'stop_container_ids' => null,
+                'stop_recovery_pending' => false,
             ]);
 
             $size = (int) instant_remote_process(
@@ -253,7 +253,7 @@ class VolumeBackupJob implements ShouldBeEncrypted, ShouldQueue
     /**
      * @param  array<int, string>  $containers
      */
-    private function archiveWithPausedContainers(string $archiveCommand, array $containers, string $stateFile): string
+    private function archiveWithStoppedContainers(string $archiveCommand, array $containers, string $stateFile): string
     {
         if ($containers === []) {
             return $archiveCommand;
@@ -263,9 +263,12 @@ class VolumeBackupJob implements ShouldBeEncrypted, ShouldQueue
         $script = "set -eu\n"
             .'state_file='.escapeshellarg($stateFile)."\n"
             .": > \"\$state_file\"\n"
-            ."cleanup() { status=\$?; trap - EXIT HUP INT TERM; while IFS= read -r container; do docker unpause \"\$container\" >/dev/null || status=1; done < \"\$state_file\"; [ \$status -eq 0 ] && rm -f \"\$state_file\"; exit \$status; }\n"
-            ."trap cleanup EXIT HUP INT TERM\n"
-            ."for container in {$containerList}; do echo \"\$container\" >> \"\$state_file\"; docker pause \"\$container\" >/dev/null; done\n"
+            ."cleanup() { status=\$?; trap - EXIT HUP INT TERM; while IFS= read -r container; do docker start \"\$container\" >/dev/null || status=1; done < \"\$state_file\"; [ \$status -eq 0 ] && rm -f \"\$state_file\"; exit \$status; }\n"
+            ."trap cleanup EXIT\n"
+            ."trap 'exit 129' HUP\n"
+            ."trap 'exit 130' INT\n"
+            ."trap 'exit 143' TERM\n"
+            ."for container in {$containerList}; do echo \"\$container\" >> \"\$state_file\"; docker stop \"\$container\" >/dev/null; done\n"
             .$archiveCommand;
 
         return 'sh -c '.escapeshellarg($script);
@@ -273,7 +276,7 @@ class VolumeBackupJob implements ShouldBeEncrypted, ShouldQueue
 
     private function recoverIncompleteBackup(ScheduledVolumeBackupExecution $execution): string
     {
-        if (! $execution->pause_recovery_pending && ! $execution->s3_cleanup_pending) {
+        if (! $execution->stop_recovery_pending && ! $execution->s3_cleanup_pending) {
             return '';
         }
 

app/Jobs/VolumeBackupRecoveryJob.php

@@ -48,7 +48,7 @@ class VolumeBackupRecoveryJob implements ShouldBeEncrypted, ShouldQueue
     {
         $execution->loadMissing('scheduledVolumeBackup.volume.resource');
 
-        if ($execution->pause_recovery_pending) {
+        if ($execution->stop_recovery_pending) {
             self::recoverContainers($execution);
         }
 
@@ -75,22 +75,22 @@ class VolumeBackupRecoveryJob implements ShouldBeEncrypted, ShouldQueue
             ->filter(fn (string $container): bool => preg_match('/^[a-f0-9]{6,64}$/i', $container) === 1)
             ->values()
             ->all();
-        $execution->update(['pause_container_ids' => $containers]);
+        $execution->update(['stop_container_ids' => $containers]);
 
         $remainingFile = $stateFile.'.remaining';
         $script = 'status=0; : > '.escapeshellarg($remainingFile).'; '
             .'if [ -f '.escapeshellarg($stateFile).' ]; then while IFS= read -r container; do '
-            .'[ -z "$container" ] && continue; paused=$(docker inspect --format \'{{.State.Paused}}\' "$container" 2>/dev/null) '
+            .'[ -z "$container" ] && continue; running=$(docker inspect --format \'{{.State.Running}}\' "$container" 2>/dev/null) '
             .'|| { echo "$container" >> '.escapeshellarg($remainingFile).'; status=1; continue; }; '
-            .'if [ "$paused" = true ] && ! docker unpause "$container" >/dev/null; then echo "$container" >> '
+            .'if [ "$running" != true ] && ! docker start "$container" >/dev/null; then echo "$container" >> '
             .escapeshellarg($remainingFile).'; status=1; fi; done < '.escapeshellarg($stateFile).'; fi; '
             .'if [ -s '.escapeshellarg($remainingFile).' ]; then mv '.escapeshellarg($remainingFile).' '.escapeshellarg($stateFile)
             .'; else rm -f '.escapeshellarg($stateFile).' '.escapeshellarg($remainingFile).'; fi; exit $status';
 
         instant_remote_process(['sh -c '.escapeshellarg($script)], $server, disableMultiplexing: true);
         $execution->update([
-            'pause_container_ids' => null,
-            'pause_recovery_pending' => false,
+            'stop_container_ids' => null,
+            'stop_recovery_pending' => false,
         ]);
     }
 
@@ -112,6 +112,6 @@ class VolumeBackupRecoveryJob implements ShouldBeEncrypted, ShouldQueue
 
     public static function stateFile(ScheduledVolumeBackupExecution $execution): string
     {
-        return '/tmp/coolify-volume-backup-'.$execution->uuid.'.paused';
+        return '/tmp/coolify-volume-backup-'.$execution->uuid.'.stopped';
     }
 }

app/Livewire/Project/Shared/Storages/VolumeBackups.php

@@ -32,7 +32,7 @@ class VolumeBackups extends Component
 
     public bool $disableLocalBackup = false;
 
-    public bool $pauseDuringBackup = false;
+    public bool $stopDuringBackup = false;
 
     public ?int $s3StorageId = null;
 
@@ -63,7 +63,7 @@ class VolumeBackups extends Component
             'enabled' => ['required', 'boolean'],
             'saveToS3' => ['required', 'boolean'],
             'disableLocalBackup' => ['required', 'boolean'],
-            'pauseDuringBackup' => ['required', 'boolean'],
+            'stopDuringBackup' => ['required', 'boolean'],
             's3StorageId' => ['nullable', 'integer'],
             'retentionAmountLocally' => ['required', 'integer', 'min:0', 'max:10000'],
             'retentionDaysLocally' => ['required', 'integer', 'min:0'],
@@ -90,7 +90,7 @@ class VolumeBackups extends Component
             $this->enabled = $this->backup->enabled;
             $this->saveToS3 = $this->backup->save_s3;
             $this->disableLocalBackup = $this->backup->disable_local_backup;
-            $this->pauseDuringBackup = $this->backup->pause_during_backup;
+            $this->stopDuringBackup = $this->backup->stop_during_backup;
             $this->s3StorageId = $this->backup->s3_storage_id ?? $this->availableS3Storages->first()?->id;
             $this->retentionAmountLocally = $this->backup->retention_amount_locally;
             $this->retentionDaysLocally = $this->backup->retention_days_locally;
@@ -188,7 +188,7 @@ class VolumeBackups extends Component
             if ($this->backup->executions()
                 ->where(fn ($query) => $query
                     ->where('status', 'running')
-                    ->orWhere('pause_recovery_pending', true)
+                    ->orWhere('stop_recovery_pending', true)
                     ->orWhere('s3_cleanup_pending', true))
                 ->exists()) {
                 $this->dispatch('error', 'Wait for the running volume backup and container recovery to finish before deleting this schedule.');
@@ -246,7 +246,7 @@ class VolumeBackups extends Component
 
         $deletedCount = $this->backup?->executions()
             ->where('status', 'failed')
-            ->where('pause_recovery_pending', false)
+            ->where('stop_recovery_pending', false)
             ->where('s3_cleanup_pending', false)
             ->where(fn ($query) => $query
                 ->whereNull('filename')
@@ -292,7 +292,7 @@ class VolumeBackups extends Component
             return false;
         }
 
-        if ($execution->status === 'running' || $execution->pause_recovery_pending || $execution->s3_cleanup_pending) {
+        if ($execution->status === 'running' || $execution->stop_recovery_pending || $execution->s3_cleanup_pending) {
             $this->dispatch('error', 'Wait for the backup and recovery operations to finish before deleting it.');
 
             return false;
@@ -369,7 +369,7 @@ class VolumeBackups extends Component
                 'enabled' => $enabled,
                 'save_s3' => $this->saveToS3,
                 'disable_local_backup' => $this->disableLocalBackup,
-                'pause_during_backup' => $this->pauseDuringBackup,
+                'stop_during_backup' => $this->stopDuringBackup,
                 's3_storage_id' => $this->saveToS3 ? $this->s3StorageId : null,
                 'retention_amount_locally' => $this->retentionAmountLocally,
                 'retention_days_locally' => $this->retentionDaysLocally,

app/Models/ScheduledVolumeBackup.php

@@ -17,7 +17,7 @@ class ScheduledVolumeBackup extends BaseModel
         'enabled',
         'save_s3',
         'disable_local_backup',
-        'pause_during_backup',
+        'stop_during_backup',
         'retention_amount_locally',
         'retention_days_locally',
         'retention_max_storage_locally',
@@ -33,7 +33,7 @@ class ScheduledVolumeBackup extends BaseModel
             'enabled' => 'boolean',
             'save_s3' => 'boolean',
             'disable_local_backup' => 'boolean',
-            'pause_during_backup' => 'boolean',
+            'stop_during_backup' => 'boolean',
             'retention_amount_locally' => 'integer',
             'retention_days_locally' => 'integer',
             'retention_max_storage_locally' => 'float',

app/Models/ScheduledVolumeBackupExecution.php

@@ -13,8 +13,8 @@ class ScheduledVolumeBackupExecution extends BaseModel
         'message',
         'size',
         'filename',
-        'pause_container_ids',
-        'pause_recovery_pending',
+        'stop_container_ids',
+        'stop_recovery_pending',
         's3_cleanup_pending',
         'finished_at',
         'local_storage_deleted',
@@ -27,8 +27,8 @@ class ScheduledVolumeBackupExecution extends BaseModel
         return [
             'size' => 'integer',
             'finished_at' => 'datetime',
-            'pause_container_ids' => 'array',
-            'pause_recovery_pending' => 'boolean',
+            'stop_container_ids' => 'array',
+            'stop_recovery_pending' => 'boolean',
             's3_cleanup_pending' => 'boolean',
             'local_storage_deleted' => 'boolean',
             's3_storage_deleted' => 'boolean',