mirror of
https://github.com/tiennm99/coolify.git
synced 2026-10-14 02:15:51 +00:00
fix(backups): enforce retention and clean up stale executions
Add `WithoutOverlapping` middleware to `DatabaseBackupJob` keyed by backup ID with timeout-based lock expiry to prevent concurrent runs. Mark long-running backup executions as failed when they exceed the stale time threshold, and add periodic retention enforcement in `CleanupInstanceStuffsJob` with cache-based throttling. Also add float casts for retention max-storage fields on `ScheduledDatabaseBackup` and comprehensive feature tests covering overlap middleware, stale detection, casts, and retention behavior.
This commit is contained in:
1 parent
968508583d
commit
ffb5045c6a
4 files changed
+319
No files matched your search
@@ -22,6 +22,7 @@ use Illuminate\Contracts\Queue\ShouldBeEncrypted;
|
||||
use Illuminate\Contracts\Queue\ShouldQueue;
|
||||
use Illuminate\Foundation\Bus\Dispatchable;
|
||||
use Illuminate\Queue\InteractsWithQueue;
|
||||
use Illuminate\Queue\Middleware\WithoutOverlapping;
|
||||
use Illuminate\Queue\SerializesModels;
|
||||
use Illuminate\Support\Facades\Log;
|
||||
use Illuminate\Support\Str;
|
||||
@@ -80,6 +81,13 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue
|
||||
$this->timeout = $backup->timeout ?? 3600;
|
||||
}
|
||||
|
||||
public function middleware(): array
|
||||
{
|
||||
$expireAfter = ($this->backup->timeout ?? 3600) + 300;
|
||||
|
||||
return [(new WithoutOverlapping('database-backup-'.$this->backup->id))->expireAfter($expireAfter)->dontRelease()];
|
||||
}
|
||||
|
||||
public function handle(): void
|
||||
{
|
||||
try {
|
||||
@@ -107,6 +115,8 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue
|
||||
throw new \Exception('Database not found?!');
|
||||
}
|
||||
|
||||
$this->markStaleExecutionsAsFailed();
|
||||
|
||||
BackupCreated::dispatch($this->team->id);
|
||||
|
||||
$status = str(data_get($this->database, 'status'));
|
||||
@@ -727,6 +737,31 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue
|
||||
return "{$helperImage}:{$latestVersion}";
|
||||
}
|
||||
|
||||
private function markStaleExecutionsAsFailed(): void
|
||||
{
|
||||
try {
|
||||
$timeoutSeconds = ($this->backup->timeout ?? 3600) * 2;
|
||||
|
||||
$staleExecutions = $this->backup->executions()
|
||||
->where('status', 'running')
|
||||
->where('created_at', '<', now()->subSeconds($timeoutSeconds))
|
||||
->get();
|
||||
|
||||
foreach ($staleExecutions as $execution) {
|
||||
$execution->update([
|
||||
'status' => 'failed',
|
||||
'message' => 'Marked as failed - backup execution exceeded maximum allowed time',
|
||||
'finished_at' => now(),
|
||||
]);
|
||||
}
|
||||
} catch (Throwable $e) {
|
||||
Log::channel('scheduled-errors')->warning('Failed to clean up stale backup executions', [
|
||||
'backup_id' => $this->backup->uuid,
|
||||
'error' => $e->getMessage(),
|
||||
]);
|
||||
}
|
||||
}
|
||||
|
||||
public function failed(?Throwable $exception): void
|
||||
{
|
||||
Log::channel('scheduled-errors')->error('DatabaseBackup permanently failed', [
|
||||
|
||||
Reference in new issue
Block a user