feat(backup): enhance backup job with S3 upload handling and notifications

- Introduced a new notification class, BackupSuccessWithS3Warning, to alert users when local backups succeed but S3 uploads fail.
- Updated DatabaseBackupJob to track local backup success and handle S3 upload errors, improving error reporting and user notifications.
- Modified ScheduledDatabaseBackupExecution model to include a new s3_uploaded boolean field for tracking S3 upload status.
- Adjusted views and validation logic to reflect changes in backup execution status and S3 handling.
- Added tests to ensure the new s3_uploaded column is correctly implemented and validated.

app/Jobs/DatabaseBackupJob.php

@@ -15,6 +15,7 @@ use App\Models\StandalonePostgresql;
 use App\Models\Team;
 use App\Notifications\Database\BackupFailed;
 use App\Notifications\Database\BackupSuccess;
+use App\Notifications\Database\BackupSuccessWithS3Warning;
 use Carbon\Carbon;
 use Illuminate\Bus\Queueable;
 use Illuminate\Contracts\Queue\ShouldBeEncrypted;
@@ -74,6 +75,7 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue
     {
         $this->onQueue('high');
         $this->timeout = $backup->timeout;
+        $this->backup_log_uuid = (string) new Cuid2;
     }
 
     public function handle(): void
@@ -298,6 +300,10 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue
                 } while ($exists);
 
                 $size = 0;
+                $localBackupSucceeded = false;
+                $s3UploadError = null;
+
+                // Step 1: Create local backup
                 try {
                     if (str($databaseType)->contains('postgres')) {
                         $this->backup_file = "/pg-dump-$database-".Carbon::now()->timestamp.'.dmp';
@@ -310,6 +316,7 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue
                             'database_name' => $database,
                             'filename' => $this->backup_location,
                             'scheduled_database_backup_id' => $this->backup->id,
+                            'local_storage_deleted' => false,
                         ]);
                         $this->backup_standalone_postgresql($database);
                     } elseif (str($databaseType)->contains('mongo')) {
@@ -330,6 +337,7 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue
                             'database_name' => $databaseName,
                             'filename' => $this->backup_location,
                             'scheduled_database_backup_id' => $this->backup->id,
+                            'local_storage_deleted' => false,
                         ]);
                         $this->backup_standalone_mongodb($database);
                     } elseif (str($databaseType)->contains('mysql')) {
@@ -343,6 +351,7 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue
                             'database_name' => $database,
                             'filename' => $this->backup_location,
                             'scheduled_database_backup_id' => $this->backup->id,
+                            'local_storage_deleted' => false,
                         ]);
                         $this->backup_standalone_mysql($database);
                     } elseif (str($databaseType)->contains('mariadb')) {
@@ -356,56 +365,77 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue
                             'database_name' => $database,
                             'filename' => $this->backup_location,
                             'scheduled_database_backup_id' => $this->backup->id,
+                            'local_storage_deleted' => false,
                         ]);
                         $this->backup_standalone_mariadb($database);
                     } else {
                         throw new \Exception('Unsupported database type');
                     }
+
                     $size = $this->calculate_size();
-                    if ($this->backup->save_s3) {
+
+                    // Verify local backup succeeded
+                    if ($size > 0) {
+                        $localBackupSucceeded = true;
+                    } else {
+                        throw new \Exception('Local backup file is empty or was not created');
+                    }
+                } catch (\Throwable $e) {
+                    // Local backup failed
+                    if ($this->backup_log) {
+                        $this->backup_log->update([
+                            'status' => 'failed',
+                            'message' => $this->error_output ?? $this->backup_output ?? $e->getMessage(),
+                            'size' => $size,
+                            'filename' => null,
+                            's3_uploaded' => null,
+                        ]);
+                    }
+                    $this->team?->notify(new BackupFailed($this->backup, $this->database, $this->error_output ?? $this->backup_output ?? $e->getMessage(), $database));
+
+                    continue;
+                }
+
+                // Step 2: Upload to S3 if enabled (independent of local backup)
+                $localStorageDeleted = false;
+                if ($this->backup->save_s3 && $localBackupSucceeded) {
+                    try {
                         $this->upload_to_s3();
 
                         // If local backup is disabled, delete the local file immediately after S3 upload
                         if ($this->backup->disable_local_backup) {
                             deleteBackupsLocally($this->backup_location, $this->server);
+                            $localStorageDeleted = true;
                         }
+                    } catch (\Throwable $e) {
+                        // S3 upload failed but local backup succeeded
+                        $s3UploadError = $e->getMessage();
                     }
+                }
 
-                    $this->team->notify(new BackupSuccess($this->backup, $this->database, $database));
+                // Step 3: Update status and send notifications based on results
+                if ($localBackupSucceeded) {
+                    $message = $this->backup_output;
+
+                    if ($s3UploadError) {
+                        $message = $message
+                            ? $message."\n\nWarning: S3 upload failed: ".$s3UploadError
+                            : 'Warning: S3 upload failed: '.$s3UploadError;
+                    }
 
                     $this->backup_log->update([
                         'status' => 'success',
-                        'message' => $this->backup_output,
+                        'message' => $message,
                         'size' => $size,
+                        's3_uploaded' => $this->backup->save_s3 ? $this->s3_uploaded : null,
+                        'local_storage_deleted' => $localStorageDeleted,
                     ]);
-                } catch (\Throwable $e) {
-                    // Check if backup actually failed or if it's just a post-backup issue
-                    $actualBackupFailed = ! $this->s3_uploaded && $this->backup->save_s3;
 
-                    if ($actualBackupFailed || $size === 0) {
-                        // Real backup failure
-                        if ($this->backup_log) {
-                            $this->backup_log->update([
-                                'status' => 'failed',
-                                'message' => $this->error_output ?? $this->backup_output ?? $e->getMessage(),
-                                'size' => $size,
-                                'filename' => null,
-                            ]);
-                        }
-                        $this->team?->notify(new BackupFailed($this->backup, $this->database, $this->error_output ?? $this->backup_output ?? $e->getMessage(), $database));
+                    // Send appropriate notification
+                    if ($s3UploadError) {
+                        $this->team->notify(new BackupSuccessWithS3Warning($this->backup, $this->database, $database, $s3UploadError));
                     } else {
-                        // Backup succeeded but post-processing failed (cleanup, notification, etc.)
-                        if ($this->backup_log) {
-                            $this->backup_log->update([
-                                'status' => 'success',
-                                'message' => $this->backup_output ? $this->backup_output."\nWarning: Post-backup cleanup encountered an issue: ".$e->getMessage() : 'Warning: '.$e->getMessage(),
-                                'size' => $size,
-                            ]);
-                        }
-                        // Send success notification since the backup itself succeeded
                         $this->team->notify(new BackupSuccess($this->backup, $this->database, $database));
-                        // Log the post-backup issue
-                        ray('Post-backup operation failed but backup was successful: '.$e->getMessage());
                     }
                 }
             }

app/Livewire/Project/Database/BackupEdit.php

@@ -208,7 +208,7 @@ class BackupEdit extends Component
 
         // Validate that disable_local_backup can only be true when S3 backup is enabled
         if ($this->backup->disable_local_backup && ! $this->backup->save_s3) {
-            throw new \Exception('Local backup can only be disabled when S3 backup is enabled.');
+            $this->backup->disable_local_backup = $this->disableLocalBackup = false;
         }
 
         $isValid = validate_cron_expression($this->backup->frequency);

app/Livewire/Project/Database/BackupExecutions.php

@@ -202,11 +202,6 @@ class BackupExecutions extends Component
 
     public function render()
     {
-        return view('livewire.project.database.backup-executions', [
-            'checkboxes' => [
-                ['id' => 'delete_backup_s3', 'label' => 'Delete the selected backup permanently from S3 Storage'],
-                // ['id' => 'delete_backup_sftp', 'label' => 'Delete the selected backup permanently from SFTP Storage'],
-            ],
-        ]);
+        return view('livewire.project.database.backup-executions');
     }
 }

app/Models/ScheduledDatabaseBackupExecution.php

@@ -8,6 +8,15 @@ class ScheduledDatabaseBackupExecution extends BaseModel
 {
     protected $guarded = [];
 
+    protected function casts(): array
+    {
+        return [
+            's3_uploaded' => 'boolean',
+            'local_storage_deleted' => 'boolean',
+            's3_storage_deleted' => 'boolean',
+        ];
+    }
+
     public function scheduledDatabaseBackup(): BelongsTo
     {
         return $this->belongsTo(ScheduledDatabaseBackup::class);

app/Notifications/Database/BackupSuccessWithS3Warning.php

@@ -0,0 +1,116 @@
+<?php
+
+namespace App\Notifications\Database;
+
+use App\Models\ScheduledDatabaseBackup;
+use App\Notifications\CustomEmailNotification;
+use App\Notifications\Dto\DiscordMessage;
+use App\Notifications\Dto\PushoverMessage;
+use App\Notifications\Dto\SlackMessage;
+use Illuminate\Notifications\Messages\MailMessage;
+
+class BackupSuccessWithS3Warning extends CustomEmailNotification
+{
+    public string $name;
+
+    public string $frequency;
+
+    public ?string $s3_storage_url = null;
+
+    public function __construct(ScheduledDatabaseBackup $backup, public $database, public $database_name, public $s3_error)
+    {
+        $this->onQueue('high');
+
+        $this->name = $database->name;
+        $this->frequency = $backup->frequency;
+
+        if ($backup->s3) {
+            $this->s3_storage_url = base_url().'/storages/'.$backup->s3->uuid;
+        }
+    }
+
+    public function via(object $notifiable): array
+    {
+        return $notifiable->getEnabledChannels('backup_failure');
+    }
+
+    public function toMail(): MailMessage
+    {
+        $mail = new MailMessage;
+        $mail->subject("Coolify: Backup succeeded locally but S3 upload failed for {$this->database->name}");
+        $mail->view('emails.backup-success-with-s3-warning', [
+            'name' => $this->name,
+            'database_name' => $this->database_name,
+            'frequency' => $this->frequency,
+            's3_error' => $this->s3_error,
+            's3_storage_url' => $this->s3_storage_url,
+        ]);
+
+        return $mail;
+    }
+
+    public function toDiscord(): DiscordMessage
+    {
+        $message = new DiscordMessage(
+            title: ':warning: Database backup succeeded locally, S3 upload failed',
+            description: "Database backup for {$this->name} (db:{$this->database_name}) was created successfully on local storage, but failed to upload to S3.",
+            color: DiscordMessage::warningColor(),
+        );
+
+        $message->addField('Frequency', $this->frequency, true);
+        $message->addField('S3 Error', $this->s3_error);
+
+        if ($this->s3_storage_url) {
+            $message->addField('S3 Storage', '[Check Configuration]('.$this->s3_storage_url.')');
+        }
+
+        return $message;
+    }
+
+    public function toTelegram(): array
+    {
+        $message = "Coolify: Database backup for {$this->name} (db:{$this->database_name}) with frequency of {$this->frequency} succeeded locally but failed to upload to S3.\n\nS3 Error:\n{$this->s3_error}";
+
+        if ($this->s3_storage_url) {
+            $message .= "\n\nCheck S3 Configuration: {$this->s3_storage_url}";
+        }
+
+        return [
+            'message' => $message,
+        ];
+    }
+
+    public function toPushover(): PushoverMessage
+    {
+        $message = "Database backup for {$this->name} (db:{$this->database_name}) was created successfully on local storage, but failed to upload to S3.<br/><br/><b>Frequency:</b> {$this->frequency}.<br/><b>S3 Error:</b> {$this->s3_error}";
+
+        if ($this->s3_storage_url) {
+            $message .= "<br/><br/><a href=\"{$this->s3_storage_url}\">Check S3 Configuration</a>";
+        }
+
+        return new PushoverMessage(
+            title: 'Database backup succeeded locally, S3 upload failed',
+            level: 'warning',
+            message: $message,
+        );
+    }
+
+    public function toSlack(): SlackMessage
+    {
+        $title = 'Database backup succeeded locally, S3 upload failed';
+        $description = "Database backup for {$this->name} (db:{$this->database_name}) was created successfully on local storage, but failed to upload to S3.";
+
+        $description .= "\n\n*Frequency:* {$this->frequency}";
+        $description .= "\n\n*S3 Error:* {$this->s3_error}";
+
+        if ($this->s3_storage_url) {
+            $description .= "\n\n*S3 Storage:* <{$this->s3_storage_url}|Check Configuration>";
+        }
+
+        return new SlackMessage(
+            title: $title,
+            description: $description,
+            color: SlackMessage::warningColor()
+        );
+    }
+}