fix(backups): enforce retention and clean up stale executions

Add `WithoutOverlapping` middleware to `DatabaseBackupJob` keyed by backup ID
with timeout-based lock expiry to prevent concurrent runs.

Mark long-running backup executions as failed when they exceed the stale
time threshold, and add periodic retention enforcement in
`CleanupInstanceStuffsJob` with cache-based throttling.

Also add float casts for retention max-storage fields on
`ScheduledDatabaseBackup` and comprehensive feature tests covering
overlap middleware, stale detection, casts, and retention behavior.

app/Jobs/CleanupInstanceStuffsJob.php

@@ -2,6 +2,7 @@
 
 namespace App\Jobs;
 
+use App\Models\ScheduledDatabaseBackup;
 use App\Models\TeamInvitation;
 use App\Models\User;
 use Illuminate\Bus\Queueable;
@@ -12,6 +13,7 @@ use Illuminate\Foundation\Bus\Dispatchable;
 use Illuminate\Queue\InteractsWithQueue;
 use Illuminate\Queue\Middleware\WithoutOverlapping;
 use Illuminate\Queue\SerializesModels;
+use Illuminate\Support\Facades\Cache;
 use Illuminate\Support\Facades\Log;
 
 class CleanupInstanceStuffsJob implements ShouldBeEncrypted, ShouldBeUnique, ShouldQueue
@@ -32,6 +34,7 @@ class CleanupInstanceStuffsJob implements ShouldBeEncrypted, ShouldBeUnique, Sho
         try {
             $this->cleanupInvitationLink();
             $this->cleanupExpiredEmailChangeRequests();
+            $this->enforceBackupRetention();
         } catch (\Throwable $e) {
             Log::error('CleanupInstanceStuffsJob failed with error: '.$e->getMessage());
         }
@@ -55,4 +58,25 @@ class CleanupInstanceStuffsJob implements ShouldBeEncrypted, ShouldBeUnique, Sho
                 'email_change_code_expires_at' => null,
             ]);
     }
+
+    private function enforceBackupRetention(): void
+    {
+        if (! Cache::add('backup-retention-enforcement', true, 1800)) {
+            return;
+        }
+
+        try {
+            $backups = ScheduledDatabaseBackup::where('enabled', true)->get();
+            foreach ($backups as $backup) {
+                try {
+                    removeOldBackups($backup);
+                } catch (\Throwable $e) {
+                    Log::warning('Failed to enforce retention for backup '.$backup->id.': '.$e->getMessage());
+                }
+            }
+        } catch (\Throwable $e) {
+            Log::error('Failed to enforce backup retention: '.$e->getMessage());
+            Cache::forget('backup-retention-enforcement');
+        }
+    }
 }

app/Jobs/DatabaseBackupJob.php

@@ -22,6 +22,7 @@ use Illuminate\Contracts\Queue\ShouldBeEncrypted;
 use Illuminate\Contracts\Queue\ShouldQueue;
 use Illuminate\Foundation\Bus\Dispatchable;
 use Illuminate\Queue\InteractsWithQueue;
+use Illuminate\Queue\Middleware\WithoutOverlapping;
 use Illuminate\Queue\SerializesModels;
 use Illuminate\Support\Facades\Log;
 use Illuminate\Support\Str;
@@ -80,6 +81,13 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue
         $this->timeout = $backup->timeout ?? 3600;
     }
 
+    public function middleware(): array
+    {
+        $expireAfter = ($this->backup->timeout ?? 3600) + 300;
+
+        return [(new WithoutOverlapping('database-backup-'.$this->backup->id))->expireAfter($expireAfter)->dontRelease()];
+    }
+
     public function handle(): void
     {
         try {
@@ -107,6 +115,8 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue
                 throw new \Exception('Database not found?!');
             }
 
+            $this->markStaleExecutionsAsFailed();
+
             BackupCreated::dispatch($this->team->id);
 
             $status = str(data_get($this->database, 'status'));
@@ -727,6 +737,31 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue
         return "{$helperImage}:{$latestVersion}";
     }
 
+    private function markStaleExecutionsAsFailed(): void
+    {
+        try {
+            $timeoutSeconds = ($this->backup->timeout ?? 3600) * 2;
+
+            $staleExecutions = $this->backup->executions()
+                ->where('status', 'running')
+                ->where('created_at', '<', now()->subSeconds($timeoutSeconds))
+                ->get();
+
+            foreach ($staleExecutions as $execution) {
+                $execution->update([
+                    'status' => 'failed',
+                    'message' => 'Marked as failed - backup execution exceeded maximum allowed time',
+                    'finished_at' => now(),
+                ]);
+            }
+        } catch (Throwable $e) {
+            Log::channel('scheduled-errors')->warning('Failed to clean up stale backup executions', [
+                'backup_id' => $this->backup->uuid,
+                'error' => $e->getMessage(),
+            ]);
+        }
+    }
+
     public function failed(?Throwable $exception): void
     {
         Log::channel('scheduled-errors')->error('DatabaseBackup permanently failed', [

app/Models/ScheduledDatabaseBackup.php

@@ -8,6 +8,14 @@ use Illuminate\Database\Eloquent\Relations\MorphTo;
 
 class ScheduledDatabaseBackup extends BaseModel
 {
+    protected function casts(): array
+    {
+        return [
+            'database_backup_retention_max_storage_locally' => 'float',
+            'database_backup_retention_max_storage_s3' => 'float',
+        ];
+    }
+
     protected $fillable = [
         'uuid',
         'team_id',