mirror of
https://github.com/tiennm99/coolify.git
synced 2026-10-15 16:14:35 +00:00
feat(backups): add scheduled persistent volume backups
This commit is contained in:
1 parent
05dc2c65f6
commit
63961e0799
36 files changed
+3506
-31
No files matched your search
@@ -4,6 +4,8 @@ namespace App\Jobs;
|
||||
|
||||
use App\Models\ScheduledDatabaseBackup;
|
||||
use App\Models\ScheduledTask;
|
||||
use App\Models\ScheduledVolumeBackup;
|
||||
use App\Models\ScheduledVolumeBackupExecution;
|
||||
use App\Models\Server;
|
||||
use App\Models\Team;
|
||||
use Cron\CronExpression;
|
||||
@@ -109,6 +111,16 @@ class ScheduledJobManager implements ShouldQueue
|
||||
]);
|
||||
}
|
||||
|
||||
try {
|
||||
$this->recoverPausedVolumeBackupContainers();
|
||||
$this->processScheduledVolumeBackups();
|
||||
} catch (\Exception $e) {
|
||||
Log::channel('scheduled-errors')->error('Failed to process scheduled volume backups', [
|
||||
'error' => $e->getMessage(),
|
||||
'trace' => $e->getTraceAsString(),
|
||||
]);
|
||||
}
|
||||
|
||||
// Process Docker cleanups - don't let failures stop the job manager
|
||||
try {
|
||||
$this->processDockerCleanups();
|
||||
@@ -341,6 +353,101 @@ class ScheduledJobManager implements ShouldQueue
|
||||
}
|
||||
}
|
||||
|
||||
private function processScheduledVolumeBackups(): void
|
||||
{
|
||||
ScheduledVolumeBackup::query()
|
||||
->with(['volume', 'team.subscription'])
|
||||
->where('enabled', true)
|
||||
->chunkById(self::CHUNK_SIZE, function ($backups): void {
|
||||
foreach ($backups as $backup) {
|
||||
$this->processScheduledVolumeBackup($backup);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
private function recoverPausedVolumeBackupContainers(): void
|
||||
{
|
||||
ScheduledVolumeBackupExecution::query()
|
||||
->where(fn (Builder $query) => $query
|
||||
->where('pause_recovery_pending', true)
|
||||
->orWhere('s3_cleanup_pending', true))
|
||||
->chunkById(self::CHUNK_SIZE, function ($executions): void {
|
||||
foreach ($executions as $execution) {
|
||||
VolumeBackupRecoveryJob::dispatch($execution);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
private function processScheduledVolumeBackup(ScheduledVolumeBackup $backup): void
|
||||
{
|
||||
try {
|
||||
$server = $backup->server();
|
||||
|
||||
if ($backup->executions()
|
||||
->where(fn (Builder $query) => $query
|
||||
->where('pause_recovery_pending', true)
|
||||
->orWhere('s3_cleanup_pending', true))
|
||||
->exists()) {
|
||||
$this->skippedCount++;
|
||||
$this->logSkip('volume_backup', 'container_recovery_pending', [
|
||||
'backup_id' => $backup->id,
|
||||
'team_id' => $backup->team_id,
|
||||
]);
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
if (! $backup->volume || ! $server) {
|
||||
$backup->delete();
|
||||
$this->skippedCount++;
|
||||
$this->logSkip('volume_backup', 'resource_deleted', [
|
||||
'backup_id' => $backup->id,
|
||||
'team_id' => $backup->team_id,
|
||||
]);
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
if (! $server->isFunctional()) {
|
||||
$this->skippedCount++;
|
||||
$this->logSkip('volume_backup', 'server_not_functional', [
|
||||
'backup_id' => $backup->id,
|
||||
'team_id' => $backup->team_id,
|
||||
'server_id' => $server->id,
|
||||
]);
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
if (isCloud() && $backup->team_id !== 0 && ! data_get($backup, 'team.subscription.stripe_invoice_paid', false)) {
|
||||
$this->skippedCount++;
|
||||
$this->logSkip('volume_backup', 'subscription_unpaid', [
|
||||
'backup_id' => $backup->id,
|
||||
'team_id' => $backup->team_id,
|
||||
'server_id' => $server->id,
|
||||
]);
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
if ($this->shouldDispatch($backup->frequency, $server, "scheduled-volume-backup:{$backup->id}")) {
|
||||
VolumeBackupJob::dispatch($backup);
|
||||
$this->dispatchedCount++;
|
||||
Log::channel('scheduled')->info('Volume backup dispatched', [
|
||||
'backup_id' => $backup->id,
|
||||
'volume_id' => $backup->local_persistent_volume_id,
|
||||
'team_id' => $backup->team_id,
|
||||
'server_id' => $server->id,
|
||||
]);
|
||||
}
|
||||
} catch (\Exception $e) {
|
||||
Log::channel('scheduled-errors')->error('Error processing volume backup', [
|
||||
'backup_id' => $backup->id,
|
||||
'error' => $e->getMessage(),
|
||||
]);
|
||||
}
|
||||
}
|
||||
|
||||
private function getBackupSkipReason(ScheduledDatabaseBackup $backup, ?Server $server): ?string
|
||||
{
|
||||
if (blank(data_get($backup, 'database'))) {
|
||||
|
||||
@@ -0,0 +1,414 @@
|
||||
<?php
|
||||
|
||||
namespace App\Jobs;
|
||||
|
||||
use App\Events\BackupCreated;
|
||||
use App\Models\ScheduledVolumeBackup;
|
||||
use App\Models\ScheduledVolumeBackupExecution;
|
||||
use App\Models\Server;
|
||||
use App\Rules\SafeWebhookUrl;
|
||||
use Carbon\Carbon;
|
||||
use Illuminate\Bus\Queueable;
|
||||
use Illuminate\Contracts\Queue\ShouldBeEncrypted;
|
||||
use Illuminate\Contracts\Queue\ShouldQueue;
|
||||
use Illuminate\Database\Eloquent\Builder;
|
||||
use Illuminate\Foundation\Bus\Dispatchable;
|
||||
use Illuminate\Queue\InteractsWithQueue;
|
||||
use Illuminate\Queue\Middleware\WithoutOverlapping;
|
||||
use Illuminate\Queue\SerializesModels;
|
||||
use Illuminate\Support\Collection;
|
||||
use Illuminate\Support\Facades\Log;
|
||||
use Throwable;
|
||||
|
||||
class VolumeBackupJob implements ShouldBeEncrypted, ShouldQueue
|
||||
{
|
||||
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
|
||||
|
||||
public int $maxExceptions = 1;
|
||||
|
||||
public int $timeout = 3600;
|
||||
|
||||
private ?ScheduledVolumeBackupExecution $execution = null;
|
||||
|
||||
public function __construct(public ScheduledVolumeBackup $backup)
|
||||
{
|
||||
$this->onQueue(crons_queue());
|
||||
$this->timeout = $backup->timeout ?? 3600;
|
||||
}
|
||||
|
||||
public function middleware(): array
|
||||
{
|
||||
return [
|
||||
(new WithoutOverlapping('volume-backup-'.$this->backup->id))
|
||||
->shared()
|
||||
->expireAfter($this->timeout + 60)
|
||||
->dontRelease(),
|
||||
];
|
||||
}
|
||||
|
||||
public static function lockKey(int $backupId): string
|
||||
{
|
||||
return 'laravel-queue-overlap:volume-backup-'.$backupId;
|
||||
}
|
||||
|
||||
public function handle(): void
|
||||
{
|
||||
$this->backup->loadMissing(['volume.resource', 'team', 's3']);
|
||||
$server = $this->backup->server();
|
||||
$volume = $this->backup->volume;
|
||||
$team = $this->backup->team;
|
||||
|
||||
if (! $server || ! $volume || ! $team) {
|
||||
throw new \RuntimeException('The volume backup resource, team, or server no longer exists.');
|
||||
}
|
||||
|
||||
$this->execution = $this->backup->executions()->create();
|
||||
BackupCreated::dispatch($team->id);
|
||||
|
||||
$backupDirectory = backup_dir().'/volumes/'.str($team->name)->slug().'-'.$team->id.'/'.$volume->uuid;
|
||||
$filename = 'volume-'.str($volume->name)->slug().'-'.Carbon::now()->timestamp.'.tar.gz';
|
||||
$backupLocation = $backupDirectory.'/'.$filename;
|
||||
$this->execution->update(['filename' => $backupLocation]);
|
||||
|
||||
try {
|
||||
$source = filled($volume->host_path) ? $volume->host_path : $volume->name;
|
||||
$containerName = 'volume-backup-'.$this->execution->uuid;
|
||||
$image = coolifyHelperImage().':'.getHelperVersion();
|
||||
$verifySourceCommand = filled($volume->host_path)
|
||||
? 'test -d '.escapeshellarg($source)
|
||||
: 'docker volume inspect '.escapeshellarg($source).' >/dev/null';
|
||||
|
||||
$archiveCommand = 'docker run --rm --name '.escapeshellarg($containerName)
|
||||
.' -v '.escapeshellarg($source.':/volume:ro')
|
||||
.' '.escapeshellarg($image)
|
||||
.' tar -czf - -C /volume . > '.escapeshellarg($backupLocation);
|
||||
|
||||
if ($this->backup->pause_during_backup) {
|
||||
$containers = $this->containersUsingVolume($source, $server);
|
||||
if ($containers !== []) {
|
||||
$this->execution->update(['pause_recovery_pending' => true]);
|
||||
$archiveCommand = $this->archiveWithPausedContainers(
|
||||
$archiveCommand,
|
||||
$containers,
|
||||
VolumeBackupRecoveryJob::stateFile($this->execution),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
instant_remote_process([
|
||||
$verifySourceCommand,
|
||||
'mkdir -p '.escapeshellarg($backupDirectory),
|
||||
$archiveCommand,
|
||||
], $server, timeout: $this->timeout, disableMultiplexing: true);
|
||||
$this->execution->update([
|
||||
'pause_container_ids' => null,
|
||||
'pause_recovery_pending' => false,
|
||||
]);
|
||||
|
||||
$size = (int) instant_remote_process(
|
||||
['du -b '.escapeshellarg($backupLocation).' | cut -f1'],
|
||||
$server,
|
||||
disableMultiplexing: true,
|
||||
);
|
||||
|
||||
if ($size <= 0) {
|
||||
throw new \RuntimeException('The volume backup archive is empty or was not created.');
|
||||
}
|
||||
|
||||
$warning = null;
|
||||
$s3Uploaded = null;
|
||||
$s3CleanupPending = false;
|
||||
$localStorageDeleted = false;
|
||||
|
||||
if ($this->backup->save_s3) {
|
||||
$s3CleanupPending = true;
|
||||
$this->execution->update(['s3_cleanup_pending' => true]);
|
||||
|
||||
try {
|
||||
$this->uploadToS3($backupLocation, $backupDirectory, $server);
|
||||
$s3Uploaded = true;
|
||||
$s3CleanupPending = false;
|
||||
} catch (Throwable $exception) {
|
||||
$s3Uploaded = false;
|
||||
$warning = 'S3 upload failed: '.$exception->getMessage();
|
||||
|
||||
try {
|
||||
VolumeBackupRecoveryJob::cleanupS3Upload($this->execution);
|
||||
$s3CleanupPending = false;
|
||||
} catch (Throwable $cleanupException) {
|
||||
VolumeBackupRecoveryJob::dispatch($this->execution);
|
||||
$warning .= ' Partial S3 upload cleanup failed: '.$cleanupException->getMessage();
|
||||
}
|
||||
}
|
||||
|
||||
if ($s3Uploaded && $this->backup->disable_local_backup) {
|
||||
try {
|
||||
deleteBackupsLocally($backupLocation, $server, throwError: true);
|
||||
$localStorageDeleted = true;
|
||||
} catch (Throwable $exception) {
|
||||
$warning = 'S3 upload succeeded, but the local archive could not be deleted: '.$exception->getMessage();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
$this->execution->update([
|
||||
'status' => 'success',
|
||||
'message' => $warning,
|
||||
'size' => $size,
|
||||
'filename' => $backupLocation,
|
||||
's3_uploaded' => $s3Uploaded,
|
||||
's3_cleanup_pending' => $s3CleanupPending,
|
||||
'local_storage_deleted' => $localStorageDeleted,
|
||||
]);
|
||||
|
||||
try {
|
||||
$this->removeExpiredBackups($server);
|
||||
} catch (Throwable $exception) {
|
||||
Log::channel('scheduled-errors')->warning('Volume backup retention cleanup failed', [
|
||||
'backup_id' => $this->backup->id,
|
||||
'execution_id' => $this->execution->id,
|
||||
'error' => $exception->getMessage(),
|
||||
]);
|
||||
}
|
||||
} catch (Throwable $exception) {
|
||||
$recoveryError = $this->recoverIncompleteBackup($this->execution);
|
||||
$archiveDeleted = false;
|
||||
|
||||
try {
|
||||
deleteBackupsLocally($backupLocation, $server, throwError: true);
|
||||
$archiveDeleted = true;
|
||||
} catch (Throwable $cleanupException) {
|
||||
$recoveryError .= ' Archive cleanup failed: '.$cleanupException->getMessage();
|
||||
}
|
||||
|
||||
$s3CleanupPending = $this->execution->fresh()->s3_cleanup_pending;
|
||||
|
||||
$this->execution->update([
|
||||
'status' => 'failed',
|
||||
'message' => $exception->getMessage().$recoveryError,
|
||||
'filename' => $archiveDeleted && ! $s3CleanupPending ? null : $backupLocation,
|
||||
'local_storage_deleted' => $archiveDeleted,
|
||||
]);
|
||||
|
||||
throw $exception;
|
||||
} finally {
|
||||
$this->execution->update(['finished_at' => now()]);
|
||||
BackupCreated::dispatch($team->id);
|
||||
}
|
||||
}
|
||||
|
||||
public function failed(?Throwable $exception): void
|
||||
{
|
||||
$execution = $this->execution ?? $this->backup->executions()
|
||||
->where('status', 'running')
|
||||
->latest('id')
|
||||
->first();
|
||||
|
||||
if ($execution) {
|
||||
$recoveryError = $this->recoverIncompleteBackup($execution);
|
||||
$server = $this->backup->server();
|
||||
$filename = $execution->filename;
|
||||
$localStorageDeleted = $execution->local_storage_deleted;
|
||||
|
||||
if ($server && filled($filename)) {
|
||||
try {
|
||||
deleteBackupsLocally($filename, $server, throwError: true);
|
||||
$localStorageDeleted = true;
|
||||
if (! $execution->fresh()->s3_cleanup_pending) {
|
||||
$filename = null;
|
||||
}
|
||||
} catch (Throwable $cleanupException) {
|
||||
$recoveryError .= ' Archive cleanup failed: '.$cleanupException->getMessage();
|
||||
}
|
||||
}
|
||||
|
||||
$execution->update([
|
||||
'status' => 'failed',
|
||||
'message' => ($exception?->getMessage() ?? 'Volume backup timed out or was terminated.').$recoveryError,
|
||||
'finished_at' => now(),
|
||||
'filename' => $filename,
|
||||
'local_storage_deleted' => $localStorageDeleted,
|
||||
]);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @return array<int, string>
|
||||
*/
|
||||
private function containersUsingVolume(string $source, Server $server): array
|
||||
{
|
||||
$output = instant_remote_process(
|
||||
["containers=\$(docker ps -q) || exit 1; for container in \$containers; do paused=\$(docker inspect --format '{{.State.Paused}}' \"\$container\") || exit 1; [ \"\$paused\" = true ] && continue; mounts=\$(docker inspect --format '{{range .Mounts}}{{println .Source}}{{println .Name}}{{end}}' \"\$container\") || exit 1; if printf '%s\\n' \"\$mounts\" | grep -Fqx -- ".escapeshellarg($source).'; then echo "$container"; fi; done'],
|
||||
$server,
|
||||
disableMultiplexing: true,
|
||||
);
|
||||
$containers = collect(preg_split('/\s+/', trim((string) $output)))
|
||||
->filter(fn (string $container): bool => preg_match('/^[a-f0-9]{6,64}$/i', $container) === 1)
|
||||
->values()
|
||||
->all();
|
||||
|
||||
return $containers;
|
||||
}
|
||||
|
||||
/**
|
||||
* @param array<int, string> $containers
|
||||
*/
|
||||
private function archiveWithPausedContainers(string $archiveCommand, array $containers, string $stateFile): string
|
||||
{
|
||||
if ($containers === []) {
|
||||
return $archiveCommand;
|
||||
}
|
||||
|
||||
$containerList = implode(' ', $containers);
|
||||
$script = "set -eu\n"
|
||||
.'state_file='.escapeshellarg($stateFile)."\n"
|
||||
.": > \"\$state_file\"\n"
|
||||
."cleanup() { status=\$?; trap - EXIT HUP INT TERM; while IFS= read -r container; do docker unpause \"\$container\" >/dev/null || status=1; done < \"\$state_file\"; [ \$status -eq 0 ] && rm -f \"\$state_file\"; exit \$status; }\n"
|
||||
."trap cleanup EXIT HUP INT TERM\n"
|
||||
."for container in {$containerList}; do echo \"\$container\" >> \"\$state_file\"; docker pause \"\$container\" >/dev/null; done\n"
|
||||
.$archiveCommand;
|
||||
|
||||
return 'sh -c '.escapeshellarg($script);
|
||||
}
|
||||
|
||||
private function recoverIncompleteBackup(ScheduledVolumeBackupExecution $execution): string
|
||||
{
|
||||
if (! $execution->pause_recovery_pending && ! $execution->s3_cleanup_pending) {
|
||||
return '';
|
||||
}
|
||||
|
||||
try {
|
||||
VolumeBackupRecoveryJob::recover($execution);
|
||||
|
||||
return '';
|
||||
} catch (Throwable $exception) {
|
||||
VolumeBackupRecoveryJob::dispatch($execution);
|
||||
|
||||
return ' Container recovery failed: '.$exception->getMessage();
|
||||
}
|
||||
}
|
||||
|
||||
private function uploadToS3(string $backupLocation, string $backupDirectory, Server $server): void
|
||||
{
|
||||
$s3 = $this->backup->s3;
|
||||
|
||||
if (! $s3) {
|
||||
$this->backup->update(['save_s3' => false, 's3_storage_id' => null]);
|
||||
|
||||
throw new \RuntimeException('The selected S3 storage no longer exists. S3 backup has been disabled.');
|
||||
}
|
||||
|
||||
$s3->testConnection(shouldSave: true);
|
||||
$containerName = 'volume-upload-'.$this->execution->uuid;
|
||||
$image = coolifyHelperImage().':'.getHelperVersion();
|
||||
$resolveOptions = collect(SafeWebhookUrl::minioClientResolveOptions($s3->endpoint))
|
||||
->map(fn (string $option): string => '--resolve '.escapeshellarg($option))
|
||||
->implode(' ');
|
||||
$resolveOptions = $resolveOptions === '' ? '' : ' '.$resolveOptions;
|
||||
|
||||
try {
|
||||
instant_remote_process([
|
||||
'docker rm -f '.escapeshellarg($containerName).' >/dev/null 2>&1 || true',
|
||||
'docker run -d --name '.escapeshellarg($containerName).' --rm -v '
|
||||
.escapeshellarg($backupLocation.':'.$backupLocation.':ro').' '.escapeshellarg($image),
|
||||
'docker exec '.escapeshellarg($containerName).' mc alias set'.$resolveOptions.' temporary '
|
||||
.escapeshellarg($s3->endpoint).' '.escapeshellarg($s3->key).' '.escapeshellarg($s3->secret),
|
||||
'docker exec '.escapeshellarg($containerName).' mc cp '.escapeshellarg($backupLocation).' '
|
||||
.escapeshellarg('temporary/'.$s3->bucket.$backupDirectory.'/'),
|
||||
], $server, timeout: $this->timeout, disableMultiplexing: true);
|
||||
} finally {
|
||||
instant_remote_process(
|
||||
['docker rm -f '.escapeshellarg($containerName)],
|
||||
$server,
|
||||
throwError: false,
|
||||
disableMultiplexing: true,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
private function removeExpiredBackups(Server $server): void
|
||||
{
|
||||
$localExecutions = $this->backup->executions()
|
||||
->where('status', 'success')
|
||||
->where('local_storage_deleted', false)
|
||||
->get();
|
||||
$localExecutions = $this->executionsOutsideRetention(
|
||||
$localExecutions,
|
||||
$this->backup->retention_amount_locally,
|
||||
$this->backup->retention_days_locally,
|
||||
$this->backup->retention_max_storage_locally,
|
||||
);
|
||||
|
||||
$filenames = $localExecutions->pluck('filename')->filter()->all();
|
||||
if ($filenames !== []) {
|
||||
deleteBackupsLocally($filenames, $server, throwError: true);
|
||||
$this->backup->executions()->whereKey($localExecutions->pluck('id')->all())
|
||||
->update(['local_storage_deleted' => true]);
|
||||
}
|
||||
|
||||
if ($this->backup->save_s3 && $this->backup->s3) {
|
||||
$s3Executions = $this->backup->executions()
|
||||
->where('status', 'success')
|
||||
->where('s3_uploaded', true)
|
||||
->where('s3_storage_deleted', false)
|
||||
->get();
|
||||
$s3Executions = $this->executionsOutsideRetention(
|
||||
$s3Executions,
|
||||
$this->backup->retention_amount_s3,
|
||||
$this->backup->retention_days_s3,
|
||||
$this->backup->retention_max_storage_s3,
|
||||
);
|
||||
|
||||
$filenames = $s3Executions->pluck('filename')->filter()->all();
|
||||
if ($filenames !== []) {
|
||||
deleteBackupsS3($filenames, $this->backup->s3);
|
||||
$this->backup->executions()->whereKey($s3Executions->pluck('id')->all())
|
||||
->update(['s3_storage_deleted' => true]);
|
||||
}
|
||||
}
|
||||
|
||||
$this->backup->executions()
|
||||
->where('local_storage_deleted', true)
|
||||
->where(function (Builder $query): void {
|
||||
$query->where('s3_storage_deleted', true)->orWhereNull('s3_uploaded');
|
||||
})
|
||||
->delete();
|
||||
}
|
||||
|
||||
private function executionsOutsideRetention(Collection $executions, int $amount, int $days, float $maxStorageGb): Collection
|
||||
{
|
||||
if ($amount === 0 && $days === 0 && $maxStorageGb == 0) {
|
||||
return collect();
|
||||
}
|
||||
|
||||
$executionsToDelete = collect();
|
||||
|
||||
if ($amount > 0) {
|
||||
$executionsToDelete = $executionsToDelete->merge($executions->skip($amount));
|
||||
}
|
||||
|
||||
if ($days > 0) {
|
||||
$oldestAllowedDate = now()->subDays($days);
|
||||
$executionsToDelete = $executionsToDelete->merge(
|
||||
$executions->filter(fn (ScheduledVolumeBackupExecution $execution): bool => $execution->created_at->isBefore($oldestAllowedDate)),
|
||||
);
|
||||
}
|
||||
|
||||
if ($maxStorageGb > 0) {
|
||||
$maxStorageBytes = $maxStorageGb * 1024 ** 3;
|
||||
$totalSize = 0;
|
||||
|
||||
foreach ($executions as $index => $execution) {
|
||||
$totalSize += (int) $execution->size;
|
||||
|
||||
if ($index > 0 && $totalSize > $maxStorageBytes) {
|
||||
$executionsToDelete = $executionsToDelete->merge($executions->slice($index));
|
||||
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return $executionsToDelete->unique('id')->values();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,117 @@
|
||||
<?php
|
||||
|
||||
namespace App\Jobs;
|
||||
|
||||
use App\Models\ScheduledVolumeBackupExecution;
|
||||
use Illuminate\Bus\Queueable;
|
||||
use Illuminate\Contracts\Queue\ShouldBeEncrypted;
|
||||
use Illuminate\Contracts\Queue\ShouldQueue;
|
||||
use Illuminate\Foundation\Bus\Dispatchable;
|
||||
use Illuminate\Queue\InteractsWithQueue;
|
||||
use Illuminate\Queue\Middleware\WithoutOverlapping;
|
||||
use Illuminate\Queue\SerializesModels;
|
||||
|
||||
class VolumeBackupRecoveryJob implements ShouldBeEncrypted, ShouldQueue
|
||||
{
|
||||
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
|
||||
|
||||
public int $tries = 20;
|
||||
|
||||
public int $backoff = 60;
|
||||
|
||||
public int $timeout = 120;
|
||||
|
||||
public function __construct(public ScheduledVolumeBackupExecution $execution)
|
||||
{
|
||||
$this->onQueue(crons_queue());
|
||||
}
|
||||
|
||||
public function middleware(): array
|
||||
{
|
||||
return [
|
||||
(new WithoutOverlapping('volume-backup-'.$this->execution->scheduled_volume_backup_id))
|
||||
->shared()
|
||||
->expireAfter(300)
|
||||
->dontRelease(),
|
||||
(new WithoutOverlapping('volume-backup-recovery-'.$this->execution->id))
|
||||
->expireAfter(300)
|
||||
->dontRelease(),
|
||||
];
|
||||
}
|
||||
|
||||
public function handle(): void
|
||||
{
|
||||
self::recover($this->execution);
|
||||
}
|
||||
|
||||
public static function recover(ScheduledVolumeBackupExecution $execution): void
|
||||
{
|
||||
$execution->loadMissing('scheduledVolumeBackup.volume.resource');
|
||||
|
||||
if ($execution->pause_recovery_pending) {
|
||||
self::recoverContainers($execution);
|
||||
}
|
||||
|
||||
if ($execution->s3_cleanup_pending) {
|
||||
self::cleanupS3Upload($execution);
|
||||
}
|
||||
}
|
||||
|
||||
private static function recoverContainers(ScheduledVolumeBackupExecution $execution): void
|
||||
{
|
||||
$server = $execution->scheduledVolumeBackup?->server();
|
||||
|
||||
if (! $server) {
|
||||
throw new \RuntimeException('The server is unavailable for container recovery.');
|
||||
}
|
||||
|
||||
$stateFile = self::stateFile($execution);
|
||||
$output = instant_remote_process(
|
||||
['cat '.escapeshellarg($stateFile).' 2>/dev/null || true'],
|
||||
$server,
|
||||
disableMultiplexing: true,
|
||||
);
|
||||
$containers = collect(preg_split('/\s+/', trim((string) $output)))
|
||||
->filter(fn (string $container): bool => preg_match('/^[a-f0-9]{6,64}$/i', $container) === 1)
|
||||
->values()
|
||||
->all();
|
||||
$execution->update(['pause_container_ids' => $containers]);
|
||||
|
||||
$remainingFile = $stateFile.'.remaining';
|
||||
$script = 'status=0; : > '.escapeshellarg($remainingFile).'; '
|
||||
.'if [ -f '.escapeshellarg($stateFile).' ]; then while IFS= read -r container; do '
|
||||
.'[ -z "$container" ] && continue; paused=$(docker inspect --format \'{{.State.Paused}}\' "$container" 2>/dev/null) '
|
||||
.'|| { echo "$container" >> '.escapeshellarg($remainingFile).'; status=1; continue; }; '
|
||||
.'if [ "$paused" = true ] && ! docker unpause "$container" >/dev/null; then echo "$container" >> '
|
||||
.escapeshellarg($remainingFile).'; status=1; fi; done < '.escapeshellarg($stateFile).'; fi; '
|
||||
.'if [ -s '.escapeshellarg($remainingFile).' ]; then mv '.escapeshellarg($remainingFile).' '.escapeshellarg($stateFile)
|
||||
.'; else rm -f '.escapeshellarg($stateFile).' '.escapeshellarg($remainingFile).'; fi; exit $status';
|
||||
|
||||
instant_remote_process(['sh -c '.escapeshellarg($script)], $server, disableMultiplexing: true);
|
||||
$execution->update([
|
||||
'pause_container_ids' => null,
|
||||
'pause_recovery_pending' => false,
|
||||
]);
|
||||
}
|
||||
|
||||
public static function cleanupS3Upload(ScheduledVolumeBackupExecution $execution): void
|
||||
{
|
||||
$execution->loadMissing('scheduledVolumeBackup.s3');
|
||||
$s3 = $execution->scheduledVolumeBackup?->s3;
|
||||
|
||||
if (! $s3 || blank($execution->filename)) {
|
||||
throw new \RuntimeException('The S3 storage or backup filename is unavailable for upload cleanup.');
|
||||
}
|
||||
|
||||
deleteBackupsS3($execution->filename, $s3);
|
||||
$execution->update([
|
||||
's3_cleanup_pending' => false,
|
||||
's3_storage_deleted' => true,
|
||||
]);
|
||||
}
|
||||
|
||||
public static function stateFile(ScheduledVolumeBackupExecution $execution): string
|
||||
{
|
||||
return '/tmp/coolify-volume-backup-'.$execution->uuid.'.paused';
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,130 @@
|
||||
<?php
|
||||
|
||||
namespace App\Livewire\Project\Application\Backup;
|
||||
|
||||
use App\Models\Application;
|
||||
use App\Models\S3Storage;
|
||||
use App\Models\ScheduledVolumeBackup;
|
||||
use Illuminate\Foundation\Auth\Access\AuthorizesRequests;
|
||||
use Illuminate\Support\Collection;
|
||||
use Livewire\Attributes\Locked;
|
||||
use Livewire\Component;
|
||||
|
||||
class Create extends Component
|
||||
{
|
||||
use AuthorizesRequests;
|
||||
|
||||
#[Locked]
|
||||
public Application $application;
|
||||
|
||||
public ?int $selectedVolumeId = null;
|
||||
|
||||
public ?int $volumeId = null;
|
||||
|
||||
public bool $volumeLocked = false;
|
||||
|
||||
public string $frequency = 'daily';
|
||||
|
||||
public bool $saveToS3 = false;
|
||||
|
||||
public ?int $s3StorageId = null;
|
||||
|
||||
public Collection $volumes;
|
||||
|
||||
public Collection $definedS3s;
|
||||
|
||||
protected function rules(): array
|
||||
{
|
||||
return [
|
||||
'volumeId' => ['required', 'integer'],
|
||||
'frequency' => ['required', 'string'],
|
||||
'saveToS3' => ['required', 'boolean'],
|
||||
's3StorageId' => ['nullable', 'integer'],
|
||||
];
|
||||
}
|
||||
|
||||
public function mount(): void
|
||||
{
|
||||
$this->authorize('view', $this->application);
|
||||
$this->volumes = $this->application->persistentStorages()->orderBy('name')->get();
|
||||
$this->definedS3s = S3Storage::ownedByCurrentTeam()->where('is_usable', true)->get();
|
||||
$this->s3StorageId = $this->definedS3s->first()?->id;
|
||||
$this->volumeLocked = $this->selectedVolumeId !== null;
|
||||
$this->volumeId = $this->selectedVolumeId ?? $this->volumes->first()?->id;
|
||||
$this->loadSelectedBackup();
|
||||
}
|
||||
|
||||
public function updatedVolumeId(): void
|
||||
{
|
||||
$this->loadSelectedBackup();
|
||||
}
|
||||
|
||||
public function submit(): void
|
||||
{
|
||||
$this->authorize('update', $this->application);
|
||||
$this->validate();
|
||||
|
||||
$volume = $this->application->persistentStorages()->whereKey($this->volumeId)->first();
|
||||
if (! $volume) {
|
||||
$this->addError('volumeId', 'Select a volume owned by this application.');
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
if (! validate_cron_expression($this->frequency)) {
|
||||
$this->addError('frequency', 'The frequency must be a valid cron or human expression.');
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
if ($this->saveToS3 && ! $this->validS3StorageExists()) {
|
||||
$this->addError('s3StorageId', 'Select a usable S3 storage owned by your team.');
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
$backup = ScheduledVolumeBackup::query()->updateOrCreate(
|
||||
['local_persistent_volume_id' => $volume->id],
|
||||
[
|
||||
'team_id' => currentTeam()->id,
|
||||
'frequency' => $this->frequency,
|
||||
'enabled' => true,
|
||||
'save_s3' => $this->saveToS3,
|
||||
's3_storage_id' => $this->saveToS3 ? $this->s3StorageId : null,
|
||||
],
|
||||
);
|
||||
|
||||
$this->dispatch('success', $backup->wasRecentlyCreated ? 'Scheduled volume backup created.' : 'Scheduled volume backup updated.');
|
||||
$this->dispatch('refreshVolumeBackups');
|
||||
$this->dispatch('close-modal');
|
||||
}
|
||||
|
||||
public function render()
|
||||
{
|
||||
return view('livewire.project.application.backup.create');
|
||||
}
|
||||
|
||||
private function loadSelectedBackup(): void
|
||||
{
|
||||
$volume = $this->volumes->firstWhere('id', $this->volumeId);
|
||||
if (! $volume) {
|
||||
return;
|
||||
}
|
||||
|
||||
$backup = $volume->scheduledBackups()->first();
|
||||
if ($backup) {
|
||||
$this->frequency = $backup->frequency;
|
||||
$this->saveToS3 = $backup->save_s3;
|
||||
$this->s3StorageId = $backup->s3_storage_id ?? $this->definedS3s->first()?->id;
|
||||
}
|
||||
}
|
||||
|
||||
private function validS3StorageExists(): bool
|
||||
{
|
||||
return $this->s3StorageId !== null
|
||||
&& S3Storage::ownedByCurrentTeam()
|
||||
->where('is_usable', true)
|
||||
->whereKey($this->s3StorageId)
|
||||
->exists();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,55 @@
|
||||
<?php
|
||||
|
||||
namespace App\Livewire\Project\Application\Backup;
|
||||
|
||||
use App\Models\Application;
|
||||
use App\Models\ScheduledVolumeBackup;
|
||||
use Illuminate\Contracts\View\View;
|
||||
use Illuminate\Foundation\Auth\Access\AuthorizesRequests;
|
||||
use Livewire\Component;
|
||||
|
||||
class Index extends Component
|
||||
{
|
||||
use AuthorizesRequests;
|
||||
|
||||
public Application $application;
|
||||
|
||||
public array $parameters;
|
||||
|
||||
protected $listeners = ['refreshVolumeBackups' => '$refresh'];
|
||||
|
||||
public function mount(): void
|
||||
{
|
||||
$this->application = $this->findApplication();
|
||||
$this->authorize('view', $this->application);
|
||||
$this->parameters = get_route_parameters();
|
||||
}
|
||||
|
||||
public function render(): View
|
||||
{
|
||||
$volumeIds = $this->application->persistentStorages()->pluck('id');
|
||||
$backups = ScheduledVolumeBackup::query()
|
||||
->with(['volume', 'latestExecution'])
|
||||
->withCount('executions')
|
||||
->whereIn('local_persistent_volume_id', $volumeIds)
|
||||
->latest()
|
||||
->get();
|
||||
|
||||
return view('livewire.project.application.backup.index', ['backups' => $backups]);
|
||||
}
|
||||
|
||||
private function findApplication(): Application
|
||||
{
|
||||
$project = currentTeam()->projects()
|
||||
->where('uuid', request()->route('project_uuid'))
|
||||
->firstOrFail();
|
||||
$environment = $project->environments()
|
||||
->where('uuid', request()->route('environment_uuid'))
|
||||
->firstOrFail();
|
||||
|
||||
return $environment->applications()
|
||||
->with('destination.server')
|
||||
->where('uuid', request()->route('application_uuid'))
|
||||
->firstOrFail();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,46 @@
|
||||
<?php
|
||||
|
||||
namespace App\Livewire\Project\Application\Backup;
|
||||
|
||||
use App\Models\Application;
|
||||
use App\Models\ScheduledVolumeBackup;
|
||||
use Illuminate\Foundation\Auth\Access\AuthorizesRequests;
|
||||
use Livewire\Component;
|
||||
|
||||
class Show extends Component
|
||||
{
|
||||
use AuthorizesRequests;
|
||||
|
||||
public Application $application;
|
||||
|
||||
public ScheduledVolumeBackup $backup;
|
||||
|
||||
public array $parameters;
|
||||
|
||||
public function mount(): void
|
||||
{
|
||||
$project = currentTeam()->projects()
|
||||
->where('uuid', request()->route('project_uuid'))
|
||||
->firstOrFail();
|
||||
$environment = $project->environments()
|
||||
->where('uuid', request()->route('environment_uuid'))
|
||||
->firstOrFail();
|
||||
$this->application = $environment->applications()
|
||||
->with('destination.server')
|
||||
->where('uuid', request()->route('application_uuid'))
|
||||
->firstOrFail();
|
||||
$this->authorize('view', $this->application);
|
||||
|
||||
$this->backup = ScheduledVolumeBackup::query()
|
||||
->with('volume')
|
||||
->where('uuid', request()->route('backup_uuid'))
|
||||
->whereIn('local_persistent_volume_id', $this->application->persistentStorages()->pluck('id'))
|
||||
->firstOrFail();
|
||||
$this->parameters = get_route_parameters();
|
||||
}
|
||||
|
||||
public function render()
|
||||
{
|
||||
return view('livewire.project.application.backup.show');
|
||||
}
|
||||
}
|
||||
@@ -217,6 +217,19 @@ class BackupEdit extends Component
|
||||
}
|
||||
}
|
||||
|
||||
public function toggleEnabled(): void
|
||||
{
|
||||
try {
|
||||
$this->authorize('manageBackups', $this->backup->database);
|
||||
|
||||
$this->backupEnabled = ! $this->backupEnabled;
|
||||
$this->backup->update(['enabled' => $this->backupEnabled]);
|
||||
$this->dispatch('success', $this->backupEnabled ? 'Backup enabled.' : 'Backup disabled.');
|
||||
} catch (\Throwable $e) {
|
||||
$this->dispatch('error', $e->getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
public function updatedS3StorageId(): void
|
||||
{
|
||||
$this->instantSave();
|
||||
|
||||
@@ -5,6 +5,7 @@ namespace App\Livewire\Project\Shared\Storages;
|
||||
use App\Models\LocalPersistentVolume;
|
||||
use App\Support\ValidationPatterns;
|
||||
use Illuminate\Foundation\Auth\Access\AuthorizesRequests;
|
||||
use Livewire\Attributes\On;
|
||||
use Livewire\Component;
|
||||
|
||||
class Show extends Component
|
||||
@@ -32,6 +33,8 @@ class Show extends Component
|
||||
|
||||
public bool $isPreviewSuffixEnabled = true;
|
||||
|
||||
public bool $hasEnabledBackup = false;
|
||||
|
||||
protected $validationAttributes = [
|
||||
'name' => 'name',
|
||||
'mountPath' => 'mount',
|
||||
@@ -81,10 +84,19 @@ class Show extends Component
|
||||
}
|
||||
}
|
||||
|
||||
public function mount()
|
||||
public function mount(): void
|
||||
{
|
||||
$this->syncData(false);
|
||||
$this->isReadOnly = $this->storage->shouldBeReadOnlyInUI();
|
||||
$this->refreshBackupStatus();
|
||||
}
|
||||
|
||||
#[On('refreshVolumeBackups')]
|
||||
public function refreshBackupStatus(): void
|
||||
{
|
||||
$this->hasEnabledBackup = $this->storage->scheduledBackups()
|
||||
->where('enabled', true)
|
||||
->exists();
|
||||
}
|
||||
|
||||
public function instantSave(): void
|
||||
@@ -115,6 +127,12 @@ class Show extends Component
|
||||
return 'The provided password is incorrect.';
|
||||
}
|
||||
|
||||
if ($this->storage->scheduledBackups()->exists()) {
|
||||
$this->dispatch('error', 'Delete this volume backup schedule and its archives before deleting the volume.');
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
$this->storage->delete();
|
||||
$this->dispatch('refreshStorages');
|
||||
|
||||
|
||||
@@ -0,0 +1,394 @@
|
||||
<?php
|
||||
|
||||
namespace App\Livewire\Project\Shared\Storages;
|
||||
|
||||
use App\Jobs\VolumeBackupJob;
|
||||
use App\Models\LocalPersistentVolume;
|
||||
use App\Models\S3Storage;
|
||||
use App\Models\ScheduledVolumeBackup;
|
||||
use Illuminate\Foundation\Auth\Access\AuthorizesRequests;
|
||||
use Illuminate\Support\Collection;
|
||||
use Illuminate\Support\Facades\Cache;
|
||||
use Livewire\Component;
|
||||
use Livewire\WithPagination;
|
||||
use Throwable;
|
||||
|
||||
class VolumeBackups extends Component
|
||||
{
|
||||
use AuthorizesRequests;
|
||||
use WithPagination;
|
||||
|
||||
public LocalPersistentVolume $storage;
|
||||
|
||||
public $resource;
|
||||
|
||||
public ?ScheduledVolumeBackup $backup = null;
|
||||
|
||||
public string $frequency = 'daily';
|
||||
|
||||
public bool $enabled = false;
|
||||
|
||||
public bool $saveToS3 = false;
|
||||
|
||||
public bool $disableLocalBackup = false;
|
||||
|
||||
public bool $pauseDuringBackup = false;
|
||||
|
||||
public ?int $s3StorageId = null;
|
||||
|
||||
public int $retentionAmountLocally = 7;
|
||||
|
||||
public int $retentionDaysLocally = 0;
|
||||
|
||||
public float $retentionMaxStorageLocally = 0;
|
||||
|
||||
public int $retentionAmountS3 = 7;
|
||||
|
||||
public int $retentionDaysS3 = 0;
|
||||
|
||||
public float $retentionMaxStorageS3 = 0;
|
||||
|
||||
public string $timezone = '';
|
||||
|
||||
public int $timeout = 3600;
|
||||
|
||||
public bool $delete_backup_s3 = false;
|
||||
|
||||
public Collection $availableS3Storages;
|
||||
|
||||
protected function rules(): array
|
||||
{
|
||||
return [
|
||||
'frequency' => ['required', 'string'],
|
||||
'enabled' => ['required', 'boolean'],
|
||||
'saveToS3' => ['required', 'boolean'],
|
||||
'disableLocalBackup' => ['required', 'boolean'],
|
||||
'pauseDuringBackup' => ['required', 'boolean'],
|
||||
's3StorageId' => ['nullable', 'integer'],
|
||||
'retentionAmountLocally' => ['required', 'integer', 'min:0', 'max:10000'],
|
||||
'retentionDaysLocally' => ['required', 'integer', 'min:0'],
|
||||
'retentionMaxStorageLocally' => ['required', 'numeric', 'min:0'],
|
||||
'retentionAmountS3' => ['required', 'integer', 'min:0', 'max:10000'],
|
||||
'retentionDaysS3' => ['required', 'integer', 'min:0'],
|
||||
'retentionMaxStorageS3' => ['required', 'numeric', 'min:0'],
|
||||
'timeout' => ['required', 'integer', 'min:60', 'max:36000'],
|
||||
];
|
||||
}
|
||||
|
||||
public function mount(): void
|
||||
{
|
||||
$this->authorize('view', $this->resource);
|
||||
$this->availableS3Storages = S3Storage::ownedByCurrentTeam()
|
||||
->where('is_usable', true)
|
||||
->get();
|
||||
$this->backup = $this->storage->scheduledBackups()->first();
|
||||
$server = $this->backup?->server() ?? data_get($this->resource, 'destination.server');
|
||||
$this->timezone = data_get($server, 'settings.server_timezone', 'Instance timezone');
|
||||
|
||||
if ($this->backup) {
|
||||
$this->frequency = $this->backup->frequency;
|
||||
$this->enabled = $this->backup->enabled;
|
||||
$this->saveToS3 = $this->backup->save_s3;
|
||||
$this->disableLocalBackup = $this->backup->disable_local_backup;
|
||||
$this->pauseDuringBackup = $this->backup->pause_during_backup;
|
||||
$this->s3StorageId = $this->backup->s3_storage_id ?? $this->availableS3Storages->first()?->id;
|
||||
$this->retentionAmountLocally = $this->backup->retention_amount_locally;
|
||||
$this->retentionDaysLocally = $this->backup->retention_days_locally;
|
||||
$this->retentionMaxStorageLocally = $this->backup->retention_max_storage_locally;
|
||||
$this->retentionAmountS3 = $this->backup->retention_amount_s3;
|
||||
$this->retentionDaysS3 = $this->backup->retention_days_s3;
|
||||
$this->retentionMaxStorageS3 = $this->backup->retention_max_storage_s3;
|
||||
$this->timeout = $this->backup->timeout;
|
||||
} else {
|
||||
$this->s3StorageId = $this->availableS3Storages->first()?->id;
|
||||
}
|
||||
}
|
||||
|
||||
public function save(): void
|
||||
{
|
||||
$this->authorize('update', $this->resource);
|
||||
|
||||
if (! $this->validateSettings()) {
|
||||
return;
|
||||
}
|
||||
|
||||
$this->backup = $this->persistBackup($this->enabled);
|
||||
$this->dispatch('success', 'Volume backup schedule saved.');
|
||||
}
|
||||
|
||||
public function instantSave(): void
|
||||
{
|
||||
$this->save();
|
||||
}
|
||||
|
||||
public function updatedS3StorageId(): void
|
||||
{
|
||||
if ($this->saveToS3) {
|
||||
$this->save();
|
||||
}
|
||||
}
|
||||
|
||||
public function toggleEnabled(): void
|
||||
{
|
||||
$this->authorize('update', $this->resource);
|
||||
|
||||
if (! $this->backup) {
|
||||
if (! $this->validateSettings()) {
|
||||
return;
|
||||
}
|
||||
|
||||
$this->enabled = true;
|
||||
$this->backup = $this->persistBackup(true);
|
||||
} else {
|
||||
$this->enabled = ! $this->enabled;
|
||||
$this->backup->update(['enabled' => $this->enabled]);
|
||||
}
|
||||
|
||||
$this->dispatch('success', $this->enabled ? 'Volume backups enabled.' : 'Volume backups disabled.');
|
||||
}
|
||||
|
||||
public function backupNow(): void
|
||||
{
|
||||
$this->authorize('update', $this->resource);
|
||||
|
||||
if (! $this->validateSettings()) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (! $this->backup) {
|
||||
$this->enabled = false;
|
||||
}
|
||||
|
||||
$this->backup = $this->persistBackup($this->enabled);
|
||||
VolumeBackupJob::dispatch($this->backup);
|
||||
$this->dispatch('success', 'Volume backup queued.');
|
||||
}
|
||||
|
||||
public function delete(?string $password = null, array $selectedActions = [])
|
||||
{
|
||||
$this->authorize('update', $this->resource);
|
||||
|
||||
if (! $password || ! verifyPasswordConfirmation($password, $this)) {
|
||||
return 'The provided password is incorrect.';
|
||||
}
|
||||
|
||||
if (! $this->backup) {
|
||||
return false;
|
||||
}
|
||||
|
||||
$lock = Cache::lock(VolumeBackupJob::lockKey($this->backup->id), $this->backup->timeout + 300);
|
||||
|
||||
if (! $lock->get()) {
|
||||
$this->dispatch('error', 'Wait for the queued or running volume backup to finish before deleting this schedule.');
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
try {
|
||||
if ($this->backup->executions()
|
||||
->where(fn ($query) => $query
|
||||
->where('status', 'running')
|
||||
->orWhere('pause_recovery_pending', true)
|
||||
->orWhere('s3_cleanup_pending', true))
|
||||
->exists()) {
|
||||
$this->dispatch('error', 'Wait for the running volume backup and container recovery to finish before deleting this schedule.');
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
$localFilenames = $this->backup->executions()
|
||||
->where('local_storage_deleted', false)
|
||||
->pluck('filename')
|
||||
->filter()
|
||||
->all();
|
||||
$server = $this->backup->server();
|
||||
|
||||
if ($localFilenames !== []) {
|
||||
if (! $server) {
|
||||
throw new \RuntimeException('The server is unavailable, so local backup archives cannot be deleted.');
|
||||
}
|
||||
|
||||
deleteBackupsLocally($localFilenames, $server, throwError: true);
|
||||
}
|
||||
|
||||
$s3Filenames = $this->backup->executions()
|
||||
->where('s3_uploaded', true)
|
||||
->where('s3_storage_deleted', false)
|
||||
->pluck('filename')
|
||||
->filter()
|
||||
->all();
|
||||
|
||||
if ($s3Filenames !== []) {
|
||||
if (! $this->backup->s3) {
|
||||
throw new \RuntimeException('The S3 storage is unavailable, so remote backup archives cannot be deleted.');
|
||||
}
|
||||
|
||||
deleteBackupsS3($s3Filenames, $this->backup->s3);
|
||||
}
|
||||
|
||||
$this->backup->delete();
|
||||
$this->backup = null;
|
||||
$this->dispatch('success', 'Volume backup schedule and archives deleted.');
|
||||
|
||||
return true;
|
||||
} catch (Throwable $exception) {
|
||||
$this->dispatch('error', 'Could not delete the backup archives: '.$exception->getMessage());
|
||||
|
||||
return false;
|
||||
} finally {
|
||||
$lock->release();
|
||||
}
|
||||
}
|
||||
|
||||
public function cleanupFailed(): void
|
||||
{
|
||||
$this->authorize('update', $this->resource);
|
||||
|
||||
$deletedCount = $this->backup?->executions()
|
||||
->where('status', 'failed')
|
||||
->where('pause_recovery_pending', false)
|
||||
->where('s3_cleanup_pending', false)
|
||||
->where(fn ($query) => $query
|
||||
->whereNull('filename')
|
||||
->orWhere('local_storage_deleted', true))
|
||||
->delete() ?? 0;
|
||||
|
||||
$this->dispatch(
|
||||
$deletedCount > 0 ? 'success' : 'info',
|
||||
$deletedCount > 0 ? 'Failed backup entries cleaned up.' : 'No safely removable failed backup entries found.',
|
||||
);
|
||||
}
|
||||
|
||||
public function cleanupDeleted(): void
|
||||
{
|
||||
$this->authorize('update', $this->resource);
|
||||
|
||||
$deletedCount = $this->backup?->executions()
|
||||
->where('local_storage_deleted', true)
|
||||
->where(fn ($query) => $query
|
||||
->where('s3_storage_deleted', true)
|
||||
->orWhereNull('s3_uploaded')
|
||||
->orWhere('s3_uploaded', false))
|
||||
->delete() ?? 0;
|
||||
|
||||
$this->dispatch(
|
||||
$deletedCount > 0 ? 'success' : 'info',
|
||||
$deletedCount > 0 ? "Cleaned up {$deletedCount} deleted backup entries." : 'No deleted backup entries found.',
|
||||
);
|
||||
}
|
||||
|
||||
public function deleteBackup(int $executionId, string $password, array $selectedActions = []): bool|string
|
||||
{
|
||||
$this->authorize('update', $this->resource);
|
||||
|
||||
if (! verifyPasswordConfirmation($password, $this)) {
|
||||
return 'The provided password is incorrect.';
|
||||
}
|
||||
|
||||
$execution = $this->backup?->executions()->whereKey($executionId)->first();
|
||||
if (! $execution) {
|
||||
$this->dispatch('error', 'Backup execution not found.');
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
if ($execution->status === 'running' || $execution->pause_recovery_pending || $execution->s3_cleanup_pending) {
|
||||
$this->dispatch('error', 'Wait for the backup and recovery operations to finish before deleting it.');
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
try {
|
||||
$server = $this->backup->server();
|
||||
if (! $execution->local_storage_deleted && filled($execution->filename)) {
|
||||
if (! $server) {
|
||||
throw new \RuntimeException('The server is unavailable.');
|
||||
}
|
||||
|
||||
deleteBackupsLocally($execution->filename, $server, throwError: true);
|
||||
}
|
||||
|
||||
if ($this->delete_backup_s3 && $execution->s3_uploaded && ! $execution->s3_storage_deleted) {
|
||||
if (! $this->backup->s3) {
|
||||
throw new \RuntimeException('The S3 storage is unavailable.');
|
||||
}
|
||||
|
||||
deleteBackupsS3($execution->filename, $this->backup->s3);
|
||||
}
|
||||
|
||||
$execution->delete();
|
||||
$this->delete_backup_s3 = false;
|
||||
$this->dispatch('success', 'Backup deleted.');
|
||||
|
||||
return true;
|
||||
} catch (Throwable $exception) {
|
||||
$this->dispatch('error', 'Failed to delete backup: '.$exception->getMessage());
|
||||
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
public function render()
|
||||
{
|
||||
$executions = $this->backup?->executions()->paginate(10);
|
||||
|
||||
return view('livewire.project.shared.storages.volume-backups', [
|
||||
'executions' => $executions ?? collect(),
|
||||
'latestExecution' => $this->backup?->executions()->first(),
|
||||
]);
|
||||
}
|
||||
|
||||
private function validateSettings(): bool
|
||||
{
|
||||
$this->validate();
|
||||
|
||||
if (! validate_cron_expression($this->frequency)) {
|
||||
$this->addError('frequency', 'The frequency must be a valid cron or human expression.');
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
if ($this->saveToS3 && ! $this->hasValidS3Storage()) {
|
||||
$this->addError('s3StorageId', 'Select a usable S3 storage owned by your team.');
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
$this->disableLocalBackup = $this->saveToS3 && $this->disableLocalBackup;
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
private function persistBackup(bool $enabled): ScheduledVolumeBackup
|
||||
{
|
||||
return ScheduledVolumeBackup::query()->updateOrCreate(
|
||||
['local_persistent_volume_id' => $this->storage->id],
|
||||
[
|
||||
'team_id' => currentTeam()->id,
|
||||
'frequency' => $this->frequency,
|
||||
'enabled' => $enabled,
|
||||
'save_s3' => $this->saveToS3,
|
||||
'disable_local_backup' => $this->disableLocalBackup,
|
||||
'pause_during_backup' => $this->pauseDuringBackup,
|
||||
's3_storage_id' => $this->saveToS3 ? $this->s3StorageId : null,
|
||||
'retention_amount_locally' => $this->retentionAmountLocally,
|
||||
'retention_days_locally' => $this->retentionDaysLocally,
|
||||
'retention_max_storage_locally' => $this->retentionMaxStorageLocally,
|
||||
'retention_amount_s3' => $this->retentionAmountS3,
|
||||
'retention_days_s3' => $this->retentionDaysS3,
|
||||
'retention_max_storage_s3' => $this->retentionMaxStorageS3,
|
||||
'timeout' => $this->timeout,
|
||||
],
|
||||
);
|
||||
}
|
||||
|
||||
private function hasValidS3Storage(): bool
|
||||
{
|
||||
return $this->s3StorageId !== null
|
||||
&& S3Storage::query()
|
||||
->whereKey($this->s3StorageId)
|
||||
->where('team_id', currentTeam()->id)
|
||||
->where('is_usable', true)
|
||||
->exists();
|
||||
}
|
||||
}
|
||||
@@ -4,6 +4,7 @@ namespace App\Livewire\Storage;
|
||||
|
||||
use App\Models\S3Storage;
|
||||
use App\Models\ScheduledDatabaseBackup;
|
||||
use App\Models\ScheduledVolumeBackup;
|
||||
use Illuminate\Auth\Access\AuthorizationException;
|
||||
use Illuminate\Foundation\Auth\Access\AuthorizesRequests;
|
||||
use Livewire\Component;
|
||||
@@ -30,7 +31,8 @@ class Show extends Component
|
||||
return $this->redirectRoute('storage.index', navigate: true);
|
||||
}
|
||||
$this->currentRoute = request()->route()->getName();
|
||||
$this->backupCount = ScheduledDatabaseBackup::where('s3_storage_id', $this->storage->id)->count();
|
||||
$this->backupCount = ScheduledDatabaseBackup::where('s3_storage_id', $this->storage->id)->count()
|
||||
+ ScheduledVolumeBackup::where('s3_storage_id', $this->storage->id)->count();
|
||||
}
|
||||
|
||||
public function delete()
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
namespace App\Models;
|
||||
|
||||
use Illuminate\Database\Eloquent\Casts\Attribute;
|
||||
use Illuminate\Database\Eloquent\Relations\HasMany;
|
||||
use Symfony\Component\Yaml\Yaml;
|
||||
|
||||
class LocalPersistentVolume extends BaseModel
|
||||
@@ -41,6 +42,11 @@ class LocalPersistentVolume extends BaseModel
|
||||
return $this->morphTo('resource');
|
||||
}
|
||||
|
||||
public function scheduledBackups(): HasMany
|
||||
{
|
||||
return $this->hasMany(ScheduledVolumeBackup::class, 'local_persistent_volume_id');
|
||||
}
|
||||
|
||||
protected function customizeName($value)
|
||||
{
|
||||
return str($value)->trim()->value;
|
||||
|
||||
@@ -69,6 +69,16 @@ class S3Storage extends BaseModel
|
||||
'save_s3' => false,
|
||||
's3_storage_id' => null,
|
||||
]);
|
||||
$volumeBackupIds = ScheduledVolumeBackup::where('s3_storage_id', $storage->id)->pluck('id');
|
||||
ScheduledVolumeBackupExecution::whereIn('scheduled_volume_backup_id', $volumeBackupIds)
|
||||
->update([
|
||||
's3_storage_deleted' => true,
|
||||
's3_cleanup_pending' => false,
|
||||
]);
|
||||
ScheduledVolumeBackup::whereKey($volumeBackupIds)->update([
|
||||
'save_s3' => false,
|
||||
's3_storage_id' => null,
|
||||
]);
|
||||
});
|
||||
}
|
||||
|
||||
@@ -101,6 +111,11 @@ class S3Storage extends BaseModel
|
||||
return $this->hasMany(ScheduledDatabaseBackup::class, 's3_storage_id');
|
||||
}
|
||||
|
||||
public function scheduledVolumeBackups()
|
||||
{
|
||||
return $this->hasMany(ScheduledVolumeBackup::class, 's3_storage_id');
|
||||
}
|
||||
|
||||
public function awsUrl()
|
||||
{
|
||||
return "{$this->endpoint}/{$this->bucket}";
|
||||
|
||||
@@ -0,0 +1,82 @@
|
||||
<?php
|
||||
|
||||
namespace App\Models;
|
||||
|
||||
use Illuminate\Database\Eloquent\Relations\BelongsTo;
|
||||
use Illuminate\Database\Eloquent\Relations\HasMany;
|
||||
use Illuminate\Database\Eloquent\Relations\HasOne;
|
||||
|
||||
class ScheduledVolumeBackup extends BaseModel
|
||||
{
|
||||
protected $fillable = [
|
||||
'uuid',
|
||||
'local_persistent_volume_id',
|
||||
'team_id',
|
||||
's3_storage_id',
|
||||
'frequency',
|
||||
'enabled',
|
||||
'save_s3',
|
||||
'disable_local_backup',
|
||||
'pause_during_backup',
|
||||
'retention_amount_locally',
|
||||
'retention_days_locally',
|
||||
'retention_max_storage_locally',
|
||||
'retention_amount_s3',
|
||||
'retention_days_s3',
|
||||
'retention_max_storage_s3',
|
||||
'timeout',
|
||||
];
|
||||
|
||||
protected function casts(): array
|
||||
{
|
||||
return [
|
||||
'enabled' => 'boolean',
|
||||
'save_s3' => 'boolean',
|
||||
'disable_local_backup' => 'boolean',
|
||||
'pause_during_backup' => 'boolean',
|
||||
'retention_amount_locally' => 'integer',
|
||||
'retention_days_locally' => 'integer',
|
||||
'retention_max_storage_locally' => 'float',
|
||||
'retention_amount_s3' => 'integer',
|
||||
'retention_days_s3' => 'integer',
|
||||
'retention_max_storage_s3' => 'float',
|
||||
'timeout' => 'integer',
|
||||
];
|
||||
}
|
||||
|
||||
public function volume(): BelongsTo
|
||||
{
|
||||
return $this->belongsTo(LocalPersistentVolume::class, 'local_persistent_volume_id');
|
||||
}
|
||||
|
||||
public function team(): BelongsTo
|
||||
{
|
||||
return $this->belongsTo(Team::class);
|
||||
}
|
||||
|
||||
public function s3(): BelongsTo
|
||||
{
|
||||
return $this->belongsTo(S3Storage::class, 's3_storage_id');
|
||||
}
|
||||
|
||||
public function executions(): HasMany
|
||||
{
|
||||
return $this->hasMany(ScheduledVolumeBackupExecution::class)->latest();
|
||||
}
|
||||
|
||||
public function latestExecution(): HasOne
|
||||
{
|
||||
return $this->hasOne(ScheduledVolumeBackupExecution::class)->latestOfMany();
|
||||
}
|
||||
|
||||
public function server(): ?Server
|
||||
{
|
||||
$resource = $this->volume?->resource;
|
||||
|
||||
if ($resource instanceof ServiceApplication || $resource instanceof ServiceDatabase) {
|
||||
return $resource->service?->server?->fresh();
|
||||
}
|
||||
|
||||
return $resource?->destination?->server?->fresh();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,43 @@
|
||||
<?php
|
||||
|
||||
namespace App\Models;
|
||||
|
||||
use Illuminate\Database\Eloquent\Relations\BelongsTo;
|
||||
|
||||
class ScheduledVolumeBackupExecution extends BaseModel
|
||||
{
|
||||
protected $fillable = [
|
||||
'uuid',
|
||||
'scheduled_volume_backup_id',
|
||||
'status',
|
||||
'message',
|
||||
'size',
|
||||
'filename',
|
||||
'pause_container_ids',
|
||||
'pause_recovery_pending',
|
||||
's3_cleanup_pending',
|
||||
'finished_at',
|
||||
'local_storage_deleted',
|
||||
's3_storage_deleted',
|
||||
's3_uploaded',
|
||||
];
|
||||
|
||||
protected function casts(): array
|
||||
{
|
||||
return [
|
||||
'size' => 'integer',
|
||||
'finished_at' => 'datetime',
|
||||
'pause_container_ids' => 'array',
|
||||
'pause_recovery_pending' => 'boolean',
|
||||
's3_cleanup_pending' => 'boolean',
|
||||
'local_storage_deleted' => 'boolean',
|
||||
's3_storage_deleted' => 'boolean',
|
||||
's3_uploaded' => 'boolean',
|
||||
];
|
||||
}
|
||||
|
||||
public function scheduledVolumeBackup(): BelongsTo
|
||||
{
|
||||
return $this->belongsTo(ScheduledVolumeBackup::class);
|
||||
}
|
||||
}
|
||||
@@ -25,6 +25,7 @@ class Checkbox extends Component
|
||||
public ?string $helper = null,
|
||||
public string|bool|null $checked = false,
|
||||
public string|bool $instantSave = false,
|
||||
public bool $live = false,
|
||||
public bool $disabled = false,
|
||||
public string $defaultClass = 'dark:border-neutral-700 text-coolgray-400 dark:bg-coolgray-100 rounded-sm cursor-pointer dark:disabled:bg-base dark:disabled:cursor-not-allowed focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-coollabs dark:focus-visible:ring-warning focus-visible:ring-offset-2 dark:focus-visible:ring-offset-base',
|
||||
public ?string $canGate = null,
|
||||
|
||||
Reference in new issue
Block a user