fix(backups): enforce retention and clean up stale executions

Add `WithoutOverlapping` middleware to `DatabaseBackupJob` keyed by backup ID
with timeout-based lock expiry to prevent concurrent runs.

Mark long-running backup executions as failed when they exceed the stale
time threshold, and add periodic retention enforcement in
`CleanupInstanceStuffsJob` with cache-based throttling.

Also add float casts for retention max-storage fields on
`ScheduledDatabaseBackup` and comprehensive feature tests covering
overlap middleware, stale detection, casts, and retention behavior.
This commit is contained in:
Andras Bacsai committed 2026-04-03 11:33:21 +02:00
1 parent 968508583d
commit ffb5045c6a
4 files changed
+319

No files matched your search

+35
View File
@@ -22,6 +22,7 @@ use Illuminate\Contracts\Queue\ShouldBeEncrypted;
use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Foundation\Bus\Dispatchable;
use Illuminate\Queue\InteractsWithQueue;
use Illuminate\Queue\Middleware\WithoutOverlapping;
use Illuminate\Queue\SerializesModels;
use Illuminate\Support\Facades\Log;
use Illuminate\Support\Str;
@@ -80,6 +81,13 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue
$this->timeout = $backup->timeout ?? 3600;
}
public function middleware(): array
{
$expireAfter = ($this->backup->timeout ?? 3600) + 300;
return [(new WithoutOverlapping('database-backup-'.$this->backup->id))->expireAfter($expireAfter)->dontRelease()];
}
public function handle(): void
{
try {
@@ -107,6 +115,8 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue
throw new \Exception('Database not found?!');
}
$this->markStaleExecutionsAsFailed();
BackupCreated::dispatch($this->team->id);
$status = str(data_get($this->database, 'status'));
@@ -727,6 +737,31 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue
return "{$helperImage}:{$latestVersion}";
}
private function markStaleExecutionsAsFailed(): void
{
try {
$timeoutSeconds = ($this->backup->timeout ?? 3600) * 2;
$staleExecutions = $this->backup->executions()
->where('status', 'running')
->where('created_at', '<', now()->subSeconds($timeoutSeconds))
->get();
foreach ($staleExecutions as $execution) {
$execution->update([
'status' => 'failed',
'message' => 'Marked as failed - backup execution exceeded maximum allowed time',
'finished_at' => now(),
]);
}
} catch (Throwable $e) {
Log::channel('scheduled-errors')->warning('Failed to clean up stale backup executions', [
'backup_id' => $this->backup->uuid,
'error' => $e->getMessage(),
]);
}
}
public function failed(?Throwable $exception): void
{
Log::channel('scheduled-errors')->error('DatabaseBackup permanently failed', [