feat(backups): add scheduled persistent volume backups

This commit is contained in:
Andras Bacsai committed 2026-07-15 15:47:57 +02:00
1 parent 05dc2c65f6
commit 63961e0799
36 files changed
+3506 -31

No files matched your search

+107
View File
@@ -4,6 +4,8 @@ namespace App\Jobs;
use App\Models\ScheduledDatabaseBackup;
use App\Models\ScheduledTask;
use App\Models\ScheduledVolumeBackup;
use App\Models\ScheduledVolumeBackupExecution;
use App\Models\Server;
use App\Models\Team;
use Cron\CronExpression;
@@ -109,6 +111,16 @@ class ScheduledJobManager implements ShouldQueue
]);
}
try {
$this->recoverPausedVolumeBackupContainers();
$this->processScheduledVolumeBackups();
} catch (\Exception $e) {
Log::channel('scheduled-errors')->error('Failed to process scheduled volume backups', [
'error' => $e->getMessage(),
'trace' => $e->getTraceAsString(),
]);
}
// Process Docker cleanups - don't let failures stop the job manager
try {
$this->processDockerCleanups();
@@ -341,6 +353,101 @@ class ScheduledJobManager implements ShouldQueue
}
}
private function processScheduledVolumeBackups(): void
{
ScheduledVolumeBackup::query()
->with(['volume', 'team.subscription'])
->where('enabled', true)
->chunkById(self::CHUNK_SIZE, function ($backups): void {
foreach ($backups as $backup) {
$this->processScheduledVolumeBackup($backup);
}
});
}
private function recoverPausedVolumeBackupContainers(): void
{
ScheduledVolumeBackupExecution::query()
->where(fn (Builder $query) => $query
->where('pause_recovery_pending', true)
->orWhere('s3_cleanup_pending', true))
->chunkById(self::CHUNK_SIZE, function ($executions): void {
foreach ($executions as $execution) {
VolumeBackupRecoveryJob::dispatch($execution);
}
});
}
private function processScheduledVolumeBackup(ScheduledVolumeBackup $backup): void
{
try {
$server = $backup->server();
if ($backup->executions()
->where(fn (Builder $query) => $query
->where('pause_recovery_pending', true)
->orWhere('s3_cleanup_pending', true))
->exists()) {
$this->skippedCount++;
$this->logSkip('volume_backup', 'container_recovery_pending', [
'backup_id' => $backup->id,
'team_id' => $backup->team_id,
]);
return;
}
if (! $backup->volume || ! $server) {
$backup->delete();
$this->skippedCount++;
$this->logSkip('volume_backup', 'resource_deleted', [
'backup_id' => $backup->id,
'team_id' => $backup->team_id,
]);
return;
}
if (! $server->isFunctional()) {
$this->skippedCount++;
$this->logSkip('volume_backup', 'server_not_functional', [
'backup_id' => $backup->id,
'team_id' => $backup->team_id,
'server_id' => $server->id,
]);
return;
}
if (isCloud() && $backup->team_id !== 0 && ! data_get($backup, 'team.subscription.stripe_invoice_paid', false)) {
$this->skippedCount++;
$this->logSkip('volume_backup', 'subscription_unpaid', [
'backup_id' => $backup->id,
'team_id' => $backup->team_id,
'server_id' => $server->id,
]);
return;
}
if ($this->shouldDispatch($backup->frequency, $server, "scheduled-volume-backup:{$backup->id}")) {
VolumeBackupJob::dispatch($backup);
$this->dispatchedCount++;
Log::channel('scheduled')->info('Volume backup dispatched', [
'backup_id' => $backup->id,
'volume_id' => $backup->local_persistent_volume_id,
'team_id' => $backup->team_id,
'server_id' => $server->id,
]);
}
} catch (\Exception $e) {
Log::channel('scheduled-errors')->error('Error processing volume backup', [
'backup_id' => $backup->id,
'error' => $e->getMessage(),
]);
}
}
private function getBackupSkipReason(ScheduledDatabaseBackup $backup, ?Server $server): ?string
{
if (blank(data_get($backup, 'database'))) {
+414
View File
@@ -0,0 +1,414 @@
<?php
namespace App\Jobs;
use App\Events\BackupCreated;
use App\Models\ScheduledVolumeBackup;
use App\Models\ScheduledVolumeBackupExecution;
use App\Models\Server;
use App\Rules\SafeWebhookUrl;
use Carbon\Carbon;
use Illuminate\Bus\Queueable;
use Illuminate\Contracts\Queue\ShouldBeEncrypted;
use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Database\Eloquent\Builder;
use Illuminate\Foundation\Bus\Dispatchable;
use Illuminate\Queue\InteractsWithQueue;
use Illuminate\Queue\Middleware\WithoutOverlapping;
use Illuminate\Queue\SerializesModels;
use Illuminate\Support\Collection;
use Illuminate\Support\Facades\Log;
use Throwable;
class VolumeBackupJob implements ShouldBeEncrypted, ShouldQueue
{
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
public int $maxExceptions = 1;
public int $timeout = 3600;
private ?ScheduledVolumeBackupExecution $execution = null;
public function __construct(public ScheduledVolumeBackup $backup)
{
$this->onQueue(crons_queue());
$this->timeout = $backup->timeout ?? 3600;
}
public function middleware(): array
{
return [
(new WithoutOverlapping('volume-backup-'.$this->backup->id))
->shared()
->expireAfter($this->timeout + 60)
->dontRelease(),
];
}
public static function lockKey(int $backupId): string
{
return 'laravel-queue-overlap:volume-backup-'.$backupId;
}
public function handle(): void
{
$this->backup->loadMissing(['volume.resource', 'team', 's3']);
$server = $this->backup->server();
$volume = $this->backup->volume;
$team = $this->backup->team;
if (! $server || ! $volume || ! $team) {
throw new \RuntimeException('The volume backup resource, team, or server no longer exists.');
}
$this->execution = $this->backup->executions()->create();
BackupCreated::dispatch($team->id);
$backupDirectory = backup_dir().'/volumes/'.str($team->name)->slug().'-'.$team->id.'/'.$volume->uuid;
$filename = 'volume-'.str($volume->name)->slug().'-'.Carbon::now()->timestamp.'.tar.gz';
$backupLocation = $backupDirectory.'/'.$filename;
$this->execution->update(['filename' => $backupLocation]);
try {
$source = filled($volume->host_path) ? $volume->host_path : $volume->name;
$containerName = 'volume-backup-'.$this->execution->uuid;
$image = coolifyHelperImage().':'.getHelperVersion();
$verifySourceCommand = filled($volume->host_path)
? 'test -d '.escapeshellarg($source)
: 'docker volume inspect '.escapeshellarg($source).' >/dev/null';
$archiveCommand = 'docker run --rm --name '.escapeshellarg($containerName)
.' -v '.escapeshellarg($source.':/volume:ro')
.' '.escapeshellarg($image)
.' tar -czf - -C /volume . > '.escapeshellarg($backupLocation);
if ($this->backup->pause_during_backup) {
$containers = $this->containersUsingVolume($source, $server);
if ($containers !== []) {
$this->execution->update(['pause_recovery_pending' => true]);
$archiveCommand = $this->archiveWithPausedContainers(
$archiveCommand,
$containers,
VolumeBackupRecoveryJob::stateFile($this->execution),
);
}
}
instant_remote_process([
$verifySourceCommand,
'mkdir -p '.escapeshellarg($backupDirectory),
$archiveCommand,
], $server, timeout: $this->timeout, disableMultiplexing: true);
$this->execution->update([
'pause_container_ids' => null,
'pause_recovery_pending' => false,
]);
$size = (int) instant_remote_process(
['du -b '.escapeshellarg($backupLocation).' | cut -f1'],
$server,
disableMultiplexing: true,
);
if ($size <= 0) {
throw new \RuntimeException('The volume backup archive is empty or was not created.');
}
$warning = null;
$s3Uploaded = null;
$s3CleanupPending = false;
$localStorageDeleted = false;
if ($this->backup->save_s3) {
$s3CleanupPending = true;
$this->execution->update(['s3_cleanup_pending' => true]);
try {
$this->uploadToS3($backupLocation, $backupDirectory, $server);
$s3Uploaded = true;
$s3CleanupPending = false;
} catch (Throwable $exception) {
$s3Uploaded = false;
$warning = 'S3 upload failed: '.$exception->getMessage();
try {
VolumeBackupRecoveryJob::cleanupS3Upload($this->execution);
$s3CleanupPending = false;
} catch (Throwable $cleanupException) {
VolumeBackupRecoveryJob::dispatch($this->execution);
$warning .= ' Partial S3 upload cleanup failed: '.$cleanupException->getMessage();
}
}
if ($s3Uploaded && $this->backup->disable_local_backup) {
try {
deleteBackupsLocally($backupLocation, $server, throwError: true);
$localStorageDeleted = true;
} catch (Throwable $exception) {
$warning = 'S3 upload succeeded, but the local archive could not be deleted: '.$exception->getMessage();
}
}
}
$this->execution->update([
'status' => 'success',
'message' => $warning,
'size' => $size,
'filename' => $backupLocation,
's3_uploaded' => $s3Uploaded,
's3_cleanup_pending' => $s3CleanupPending,
'local_storage_deleted' => $localStorageDeleted,
]);
try {
$this->removeExpiredBackups($server);
} catch (Throwable $exception) {
Log::channel('scheduled-errors')->warning('Volume backup retention cleanup failed', [
'backup_id' => $this->backup->id,
'execution_id' => $this->execution->id,
'error' => $exception->getMessage(),
]);
}
} catch (Throwable $exception) {
$recoveryError = $this->recoverIncompleteBackup($this->execution);
$archiveDeleted = false;
try {
deleteBackupsLocally($backupLocation, $server, throwError: true);
$archiveDeleted = true;
} catch (Throwable $cleanupException) {
$recoveryError .= ' Archive cleanup failed: '.$cleanupException->getMessage();
}
$s3CleanupPending = $this->execution->fresh()->s3_cleanup_pending;
$this->execution->update([
'status' => 'failed',
'message' => $exception->getMessage().$recoveryError,
'filename' => $archiveDeleted && ! $s3CleanupPending ? null : $backupLocation,
'local_storage_deleted' => $archiveDeleted,
]);
throw $exception;
} finally {
$this->execution->update(['finished_at' => now()]);
BackupCreated::dispatch($team->id);
}
}
public function failed(?Throwable $exception): void
{
$execution = $this->execution ?? $this->backup->executions()
->where('status', 'running')
->latest('id')
->first();
if ($execution) {
$recoveryError = $this->recoverIncompleteBackup($execution);
$server = $this->backup->server();
$filename = $execution->filename;
$localStorageDeleted = $execution->local_storage_deleted;
if ($server && filled($filename)) {
try {
deleteBackupsLocally($filename, $server, throwError: true);
$localStorageDeleted = true;
if (! $execution->fresh()->s3_cleanup_pending) {
$filename = null;
}
} catch (Throwable $cleanupException) {
$recoveryError .= ' Archive cleanup failed: '.$cleanupException->getMessage();
}
}
$execution->update([
'status' => 'failed',
'message' => ($exception?->getMessage() ?? 'Volume backup timed out or was terminated.').$recoveryError,
'finished_at' => now(),
'filename' => $filename,
'local_storage_deleted' => $localStorageDeleted,
]);
}
}
/**
* @return array<int, string>
*/
private function containersUsingVolume(string $source, Server $server): array
{
$output = instant_remote_process(
["containers=\$(docker ps -q) || exit 1; for container in \$containers; do paused=\$(docker inspect --format '{{.State.Paused}}' \"\$container\") || exit 1; [ \"\$paused\" = true ] && continue; mounts=\$(docker inspect --format '{{range .Mounts}}{{println .Source}}{{println .Name}}{{end}}' \"\$container\") || exit 1; if printf '%s\\n' \"\$mounts\" | grep -Fqx -- ".escapeshellarg($source).'; then echo "$container"; fi; done'],
$server,
disableMultiplexing: true,
);
$containers = collect(preg_split('/\s+/', trim((string) $output)))
->filter(fn (string $container): bool => preg_match('/^[a-f0-9]{6,64}$/i', $container) === 1)
->values()
->all();
return $containers;
}
/**
* @param array<int, string> $containers
*/
private function archiveWithPausedContainers(string $archiveCommand, array $containers, string $stateFile): string
{
if ($containers === []) {
return $archiveCommand;
}
$containerList = implode(' ', $containers);
$script = "set -eu\n"
.'state_file='.escapeshellarg($stateFile)."\n"
.": > \"\$state_file\"\n"
."cleanup() { status=\$?; trap - EXIT HUP INT TERM; while IFS= read -r container; do docker unpause \"\$container\" >/dev/null || status=1; done < \"\$state_file\"; [ \$status -eq 0 ] && rm -f \"\$state_file\"; exit \$status; }\n"
."trap cleanup EXIT HUP INT TERM\n"
."for container in {$containerList}; do echo \"\$container\" >> \"\$state_file\"; docker pause \"\$container\" >/dev/null; done\n"
.$archiveCommand;
return 'sh -c '.escapeshellarg($script);
}
private function recoverIncompleteBackup(ScheduledVolumeBackupExecution $execution): string
{
if (! $execution->pause_recovery_pending && ! $execution->s3_cleanup_pending) {
return '';
}
try {
VolumeBackupRecoveryJob::recover($execution);
return '';
} catch (Throwable $exception) {
VolumeBackupRecoveryJob::dispatch($execution);
return ' Container recovery failed: '.$exception->getMessage();
}
}
private function uploadToS3(string $backupLocation, string $backupDirectory, Server $server): void
{
$s3 = $this->backup->s3;
if (! $s3) {
$this->backup->update(['save_s3' => false, 's3_storage_id' => null]);
throw new \RuntimeException('The selected S3 storage no longer exists. S3 backup has been disabled.');
}
$s3->testConnection(shouldSave: true);
$containerName = 'volume-upload-'.$this->execution->uuid;
$image = coolifyHelperImage().':'.getHelperVersion();
$resolveOptions = collect(SafeWebhookUrl::minioClientResolveOptions($s3->endpoint))
->map(fn (string $option): string => '--resolve '.escapeshellarg($option))
->implode(' ');
$resolveOptions = $resolveOptions === '' ? '' : ' '.$resolveOptions;
try {
instant_remote_process([
'docker rm -f '.escapeshellarg($containerName).' >/dev/null 2>&1 || true',
'docker run -d --name '.escapeshellarg($containerName).' --rm -v '
.escapeshellarg($backupLocation.':'.$backupLocation.':ro').' '.escapeshellarg($image),
'docker exec '.escapeshellarg($containerName).' mc alias set'.$resolveOptions.' temporary '
.escapeshellarg($s3->endpoint).' '.escapeshellarg($s3->key).' '.escapeshellarg($s3->secret),
'docker exec '.escapeshellarg($containerName).' mc cp '.escapeshellarg($backupLocation).' '
.escapeshellarg('temporary/'.$s3->bucket.$backupDirectory.'/'),
], $server, timeout: $this->timeout, disableMultiplexing: true);
} finally {
instant_remote_process(
['docker rm -f '.escapeshellarg($containerName)],
$server,
throwError: false,
disableMultiplexing: true,
);
}
}
private function removeExpiredBackups(Server $server): void
{
$localExecutions = $this->backup->executions()
->where('status', 'success')
->where('local_storage_deleted', false)
->get();
$localExecutions = $this->executionsOutsideRetention(
$localExecutions,
$this->backup->retention_amount_locally,
$this->backup->retention_days_locally,
$this->backup->retention_max_storage_locally,
);
$filenames = $localExecutions->pluck('filename')->filter()->all();
if ($filenames !== []) {
deleteBackupsLocally($filenames, $server, throwError: true);
$this->backup->executions()->whereKey($localExecutions->pluck('id')->all())
->update(['local_storage_deleted' => true]);
}
if ($this->backup->save_s3 && $this->backup->s3) {
$s3Executions = $this->backup->executions()
->where('status', 'success')
->where('s3_uploaded', true)
->where('s3_storage_deleted', false)
->get();
$s3Executions = $this->executionsOutsideRetention(
$s3Executions,
$this->backup->retention_amount_s3,
$this->backup->retention_days_s3,
$this->backup->retention_max_storage_s3,
);
$filenames = $s3Executions->pluck('filename')->filter()->all();
if ($filenames !== []) {
deleteBackupsS3($filenames, $this->backup->s3);
$this->backup->executions()->whereKey($s3Executions->pluck('id')->all())
->update(['s3_storage_deleted' => true]);
}
}
$this->backup->executions()
->where('local_storage_deleted', true)
->where(function (Builder $query): void {
$query->where('s3_storage_deleted', true)->orWhereNull('s3_uploaded');
})
->delete();
}
private function executionsOutsideRetention(Collection $executions, int $amount, int $days, float $maxStorageGb): Collection
{
if ($amount === 0 && $days === 0 && $maxStorageGb == 0) {
return collect();
}
$executionsToDelete = collect();
if ($amount > 0) {
$executionsToDelete = $executionsToDelete->merge($executions->skip($amount));
}
if ($days > 0) {
$oldestAllowedDate = now()->subDays($days);
$executionsToDelete = $executionsToDelete->merge(
$executions->filter(fn (ScheduledVolumeBackupExecution $execution): bool => $execution->created_at->isBefore($oldestAllowedDate)),
);
}
if ($maxStorageGb > 0) {
$maxStorageBytes = $maxStorageGb * 1024 ** 3;
$totalSize = 0;
foreach ($executions as $index => $execution) {
$totalSize += (int) $execution->size;
if ($index > 0 && $totalSize > $maxStorageBytes) {
$executionsToDelete = $executionsToDelete->merge($executions->slice($index));
break;
}
}
}
return $executionsToDelete->unique('id')->values();
}
}
+117
View File
@@ -0,0 +1,117 @@
<?php
namespace App\Jobs;
use App\Models\ScheduledVolumeBackupExecution;
use Illuminate\Bus\Queueable;
use Illuminate\Contracts\Queue\ShouldBeEncrypted;
use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Foundation\Bus\Dispatchable;
use Illuminate\Queue\InteractsWithQueue;
use Illuminate\Queue\Middleware\WithoutOverlapping;
use Illuminate\Queue\SerializesModels;
class VolumeBackupRecoveryJob implements ShouldBeEncrypted, ShouldQueue
{
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
public int $tries = 20;
public int $backoff = 60;
public int $timeout = 120;
public function __construct(public ScheduledVolumeBackupExecution $execution)
{
$this->onQueue(crons_queue());
}
public function middleware(): array
{
return [
(new WithoutOverlapping('volume-backup-'.$this->execution->scheduled_volume_backup_id))
->shared()
->expireAfter(300)
->dontRelease(),
(new WithoutOverlapping('volume-backup-recovery-'.$this->execution->id))
->expireAfter(300)
->dontRelease(),
];
}
public function handle(): void
{
self::recover($this->execution);
}
public static function recover(ScheduledVolumeBackupExecution $execution): void
{
$execution->loadMissing('scheduledVolumeBackup.volume.resource');
if ($execution->pause_recovery_pending) {
self::recoverContainers($execution);
}
if ($execution->s3_cleanup_pending) {
self::cleanupS3Upload($execution);
}
}
private static function recoverContainers(ScheduledVolumeBackupExecution $execution): void
{
$server = $execution->scheduledVolumeBackup?->server();
if (! $server) {
throw new \RuntimeException('The server is unavailable for container recovery.');
}
$stateFile = self::stateFile($execution);
$output = instant_remote_process(
['cat '.escapeshellarg($stateFile).' 2>/dev/null || true'],
$server,
disableMultiplexing: true,
);
$containers = collect(preg_split('/\s+/', trim((string) $output)))
->filter(fn (string $container): bool => preg_match('/^[a-f0-9]{6,64}$/i', $container) === 1)
->values()
->all();
$execution->update(['pause_container_ids' => $containers]);
$remainingFile = $stateFile.'.remaining';
$script = 'status=0; : > '.escapeshellarg($remainingFile).'; '
.'if [ -f '.escapeshellarg($stateFile).' ]; then while IFS= read -r container; do '
.'[ -z "$container" ] && continue; paused=$(docker inspect --format \'{{.State.Paused}}\' "$container" 2>/dev/null) '
.'|| { echo "$container" >> '.escapeshellarg($remainingFile).'; status=1; continue; }; '
.'if [ "$paused" = true ] && ! docker unpause "$container" >/dev/null; then echo "$container" >> '
.escapeshellarg($remainingFile).'; status=1; fi; done < '.escapeshellarg($stateFile).'; fi; '
.'if [ -s '.escapeshellarg($remainingFile).' ]; then mv '.escapeshellarg($remainingFile).' '.escapeshellarg($stateFile)
.'; else rm -f '.escapeshellarg($stateFile).' '.escapeshellarg($remainingFile).'; fi; exit $status';
instant_remote_process(['sh -c '.escapeshellarg($script)], $server, disableMultiplexing: true);
$execution->update([
'pause_container_ids' => null,
'pause_recovery_pending' => false,
]);
}
public static function cleanupS3Upload(ScheduledVolumeBackupExecution $execution): void
{
$execution->loadMissing('scheduledVolumeBackup.s3');
$s3 = $execution->scheduledVolumeBackup?->s3;
if (! $s3 || blank($execution->filename)) {
throw new \RuntimeException('The S3 storage or backup filename is unavailable for upload cleanup.');
}
deleteBackupsS3($execution->filename, $s3);
$execution->update([
's3_cleanup_pending' => false,
's3_storage_deleted' => true,
]);
}
public static function stateFile(ScheduledVolumeBackupExecution $execution): string
{
return '/tmp/coolify-volume-backup-'.$execution->uuid.'.paused';
}
}
@@ -0,0 +1,130 @@
<?php
namespace App\Livewire\Project\Application\Backup;
use App\Models\Application;
use App\Models\S3Storage;
use App\Models\ScheduledVolumeBackup;
use Illuminate\Foundation\Auth\Access\AuthorizesRequests;
use Illuminate\Support\Collection;
use Livewire\Attributes\Locked;
use Livewire\Component;
class Create extends Component
{
use AuthorizesRequests;
#[Locked]
public Application $application;
public ?int $selectedVolumeId = null;
public ?int $volumeId = null;
public bool $volumeLocked = false;
public string $frequency = 'daily';
public bool $saveToS3 = false;
public ?int $s3StorageId = null;
public Collection $volumes;
public Collection $definedS3s;
protected function rules(): array
{
return [
'volumeId' => ['required', 'integer'],
'frequency' => ['required', 'string'],
'saveToS3' => ['required', 'boolean'],
's3StorageId' => ['nullable', 'integer'],
];
}
public function mount(): void
{
$this->authorize('view', $this->application);
$this->volumes = $this->application->persistentStorages()->orderBy('name')->get();
$this->definedS3s = S3Storage::ownedByCurrentTeam()->where('is_usable', true)->get();
$this->s3StorageId = $this->definedS3s->first()?->id;
$this->volumeLocked = $this->selectedVolumeId !== null;
$this->volumeId = $this->selectedVolumeId ?? $this->volumes->first()?->id;
$this->loadSelectedBackup();
}
public function updatedVolumeId(): void
{
$this->loadSelectedBackup();
}
public function submit(): void
{
$this->authorize('update', $this->application);
$this->validate();
$volume = $this->application->persistentStorages()->whereKey($this->volumeId)->first();
if (! $volume) {
$this->addError('volumeId', 'Select a volume owned by this application.');
return;
}
if (! validate_cron_expression($this->frequency)) {
$this->addError('frequency', 'The frequency must be a valid cron or human expression.');
return;
}
if ($this->saveToS3 && ! $this->validS3StorageExists()) {
$this->addError('s3StorageId', 'Select a usable S3 storage owned by your team.');
return;
}
$backup = ScheduledVolumeBackup::query()->updateOrCreate(
['local_persistent_volume_id' => $volume->id],
[
'team_id' => currentTeam()->id,
'frequency' => $this->frequency,
'enabled' => true,
'save_s3' => $this->saveToS3,
's3_storage_id' => $this->saveToS3 ? $this->s3StorageId : null,
],
);
$this->dispatch('success', $backup->wasRecentlyCreated ? 'Scheduled volume backup created.' : 'Scheduled volume backup updated.');
$this->dispatch('refreshVolumeBackups');
$this->dispatch('close-modal');
}
public function render()
{
return view('livewire.project.application.backup.create');
}
private function loadSelectedBackup(): void
{
$volume = $this->volumes->firstWhere('id', $this->volumeId);
if (! $volume) {
return;
}
$backup = $volume->scheduledBackups()->first();
if ($backup) {
$this->frequency = $backup->frequency;
$this->saveToS3 = $backup->save_s3;
$this->s3StorageId = $backup->s3_storage_id ?? $this->definedS3s->first()?->id;
}
}
private function validS3StorageExists(): bool
{
return $this->s3StorageId !== null
&& S3Storage::ownedByCurrentTeam()
->where('is_usable', true)
->whereKey($this->s3StorageId)
->exists();
}
}
@@ -0,0 +1,55 @@
<?php
namespace App\Livewire\Project\Application\Backup;
use App\Models\Application;
use App\Models\ScheduledVolumeBackup;
use Illuminate\Contracts\View\View;
use Illuminate\Foundation\Auth\Access\AuthorizesRequests;
use Livewire\Component;
class Index extends Component
{
use AuthorizesRequests;
public Application $application;
public array $parameters;
protected $listeners = ['refreshVolumeBackups' => '$refresh'];
public function mount(): void
{
$this->application = $this->findApplication();
$this->authorize('view', $this->application);
$this->parameters = get_route_parameters();
}
public function render(): View
{
$volumeIds = $this->application->persistentStorages()->pluck('id');
$backups = ScheduledVolumeBackup::query()
->with(['volume', 'latestExecution'])
->withCount('executions')
->whereIn('local_persistent_volume_id', $volumeIds)
->latest()
->get();
return view('livewire.project.application.backup.index', ['backups' => $backups]);
}
private function findApplication(): Application
{
$project = currentTeam()->projects()
->where('uuid', request()->route('project_uuid'))
->firstOrFail();
$environment = $project->environments()
->where('uuid', request()->route('environment_uuid'))
->firstOrFail();
return $environment->applications()
->with('destination.server')
->where('uuid', request()->route('application_uuid'))
->firstOrFail();
}
}
@@ -0,0 +1,46 @@
<?php
namespace App\Livewire\Project\Application\Backup;
use App\Models\Application;
use App\Models\ScheduledVolumeBackup;
use Illuminate\Foundation\Auth\Access\AuthorizesRequests;
use Livewire\Component;
class Show extends Component
{
use AuthorizesRequests;
public Application $application;
public ScheduledVolumeBackup $backup;
public array $parameters;
public function mount(): void
{
$project = currentTeam()->projects()
->where('uuid', request()->route('project_uuid'))
->firstOrFail();
$environment = $project->environments()
->where('uuid', request()->route('environment_uuid'))
->firstOrFail();
$this->application = $environment->applications()
->with('destination.server')
->where('uuid', request()->route('application_uuid'))
->firstOrFail();
$this->authorize('view', $this->application);
$this->backup = ScheduledVolumeBackup::query()
->with('volume')
->where('uuid', request()->route('backup_uuid'))
->whereIn('local_persistent_volume_id', $this->application->persistentStorages()->pluck('id'))
->firstOrFail();
$this->parameters = get_route_parameters();
}
public function render()
{
return view('livewire.project.application.backup.show');
}
}
@@ -217,6 +217,19 @@ class BackupEdit extends Component
}
}
public function toggleEnabled(): void
{
try {
$this->authorize('manageBackups', $this->backup->database);
$this->backupEnabled = ! $this->backupEnabled;
$this->backup->update(['enabled' => $this->backupEnabled]);
$this->dispatch('success', $this->backupEnabled ? 'Backup enabled.' : 'Backup disabled.');
} catch (\Throwable $e) {
$this->dispatch('error', $e->getMessage());
}
}
public function updatedS3StorageId(): void
{
$this->instantSave();
+19 -1
View File
@@ -5,6 +5,7 @@ namespace App\Livewire\Project\Shared\Storages;
use App\Models\LocalPersistentVolume;
use App\Support\ValidationPatterns;
use Illuminate\Foundation\Auth\Access\AuthorizesRequests;
use Livewire\Attributes\On;
use Livewire\Component;
class Show extends Component
@@ -32,6 +33,8 @@ class Show extends Component
public bool $isPreviewSuffixEnabled = true;
public bool $hasEnabledBackup = false;
protected $validationAttributes = [
'name' => 'name',
'mountPath' => 'mount',
@@ -81,10 +84,19 @@ class Show extends Component
}
}
public function mount()
public function mount(): void
{
$this->syncData(false);
$this->isReadOnly = $this->storage->shouldBeReadOnlyInUI();
$this->refreshBackupStatus();
}
#[On('refreshVolumeBackups')]
public function refreshBackupStatus(): void
{
$this->hasEnabledBackup = $this->storage->scheduledBackups()
->where('enabled', true)
->exists();
}
public function instantSave(): void
@@ -115,6 +127,12 @@ class Show extends Component
return 'The provided password is incorrect.';
}
if ($this->storage->scheduledBackups()->exists()) {
$this->dispatch('error', 'Delete this volume backup schedule and its archives before deleting the volume.');
return false;
}
$this->storage->delete();
$this->dispatch('refreshStorages');
@@ -0,0 +1,394 @@
<?php
namespace App\Livewire\Project\Shared\Storages;
use App\Jobs\VolumeBackupJob;
use App\Models\LocalPersistentVolume;
use App\Models\S3Storage;
use App\Models\ScheduledVolumeBackup;
use Illuminate\Foundation\Auth\Access\AuthorizesRequests;
use Illuminate\Support\Collection;
use Illuminate\Support\Facades\Cache;
use Livewire\Component;
use Livewire\WithPagination;
use Throwable;
class VolumeBackups extends Component
{
use AuthorizesRequests;
use WithPagination;
public LocalPersistentVolume $storage;
public $resource;
public ?ScheduledVolumeBackup $backup = null;
public string $frequency = 'daily';
public bool $enabled = false;
public bool $saveToS3 = false;
public bool $disableLocalBackup = false;
public bool $pauseDuringBackup = false;
public ?int $s3StorageId = null;
public int $retentionAmountLocally = 7;
public int $retentionDaysLocally = 0;
public float $retentionMaxStorageLocally = 0;
public int $retentionAmountS3 = 7;
public int $retentionDaysS3 = 0;
public float $retentionMaxStorageS3 = 0;
public string $timezone = '';
public int $timeout = 3600;
public bool $delete_backup_s3 = false;
public Collection $availableS3Storages;
protected function rules(): array
{
return [
'frequency' => ['required', 'string'],
'enabled' => ['required', 'boolean'],
'saveToS3' => ['required', 'boolean'],
'disableLocalBackup' => ['required', 'boolean'],
'pauseDuringBackup' => ['required', 'boolean'],
's3StorageId' => ['nullable', 'integer'],
'retentionAmountLocally' => ['required', 'integer', 'min:0', 'max:10000'],
'retentionDaysLocally' => ['required', 'integer', 'min:0'],
'retentionMaxStorageLocally' => ['required', 'numeric', 'min:0'],
'retentionAmountS3' => ['required', 'integer', 'min:0', 'max:10000'],
'retentionDaysS3' => ['required', 'integer', 'min:0'],
'retentionMaxStorageS3' => ['required', 'numeric', 'min:0'],
'timeout' => ['required', 'integer', 'min:60', 'max:36000'],
];
}
public function mount(): void
{
$this->authorize('view', $this->resource);
$this->availableS3Storages = S3Storage::ownedByCurrentTeam()
->where('is_usable', true)
->get();
$this->backup = $this->storage->scheduledBackups()->first();
$server = $this->backup?->server() ?? data_get($this->resource, 'destination.server');
$this->timezone = data_get($server, 'settings.server_timezone', 'Instance timezone');
if ($this->backup) {
$this->frequency = $this->backup->frequency;
$this->enabled = $this->backup->enabled;
$this->saveToS3 = $this->backup->save_s3;
$this->disableLocalBackup = $this->backup->disable_local_backup;
$this->pauseDuringBackup = $this->backup->pause_during_backup;
$this->s3StorageId = $this->backup->s3_storage_id ?? $this->availableS3Storages->first()?->id;
$this->retentionAmountLocally = $this->backup->retention_amount_locally;
$this->retentionDaysLocally = $this->backup->retention_days_locally;
$this->retentionMaxStorageLocally = $this->backup->retention_max_storage_locally;
$this->retentionAmountS3 = $this->backup->retention_amount_s3;
$this->retentionDaysS3 = $this->backup->retention_days_s3;
$this->retentionMaxStorageS3 = $this->backup->retention_max_storage_s3;
$this->timeout = $this->backup->timeout;
} else {
$this->s3StorageId = $this->availableS3Storages->first()?->id;
}
}
public function save(): void
{
$this->authorize('update', $this->resource);
if (! $this->validateSettings()) {
return;
}
$this->backup = $this->persistBackup($this->enabled);
$this->dispatch('success', 'Volume backup schedule saved.');
}
public function instantSave(): void
{
$this->save();
}
public function updatedS3StorageId(): void
{
if ($this->saveToS3) {
$this->save();
}
}
public function toggleEnabled(): void
{
$this->authorize('update', $this->resource);
if (! $this->backup) {
if (! $this->validateSettings()) {
return;
}
$this->enabled = true;
$this->backup = $this->persistBackup(true);
} else {
$this->enabled = ! $this->enabled;
$this->backup->update(['enabled' => $this->enabled]);
}
$this->dispatch('success', $this->enabled ? 'Volume backups enabled.' : 'Volume backups disabled.');
}
public function backupNow(): void
{
$this->authorize('update', $this->resource);
if (! $this->validateSettings()) {
return;
}
if (! $this->backup) {
$this->enabled = false;
}
$this->backup = $this->persistBackup($this->enabled);
VolumeBackupJob::dispatch($this->backup);
$this->dispatch('success', 'Volume backup queued.');
}
public function delete(?string $password = null, array $selectedActions = [])
{
$this->authorize('update', $this->resource);
if (! $password || ! verifyPasswordConfirmation($password, $this)) {
return 'The provided password is incorrect.';
}
if (! $this->backup) {
return false;
}
$lock = Cache::lock(VolumeBackupJob::lockKey($this->backup->id), $this->backup->timeout + 300);
if (! $lock->get()) {
$this->dispatch('error', 'Wait for the queued or running volume backup to finish before deleting this schedule.');
return false;
}
try {
if ($this->backup->executions()
->where(fn ($query) => $query
->where('status', 'running')
->orWhere('pause_recovery_pending', true)
->orWhere('s3_cleanup_pending', true))
->exists()) {
$this->dispatch('error', 'Wait for the running volume backup and container recovery to finish before deleting this schedule.');
return false;
}
$localFilenames = $this->backup->executions()
->where('local_storage_deleted', false)
->pluck('filename')
->filter()
->all();
$server = $this->backup->server();
if ($localFilenames !== []) {
if (! $server) {
throw new \RuntimeException('The server is unavailable, so local backup archives cannot be deleted.');
}
deleteBackupsLocally($localFilenames, $server, throwError: true);
}
$s3Filenames = $this->backup->executions()
->where('s3_uploaded', true)
->where('s3_storage_deleted', false)
->pluck('filename')
->filter()
->all();
if ($s3Filenames !== []) {
if (! $this->backup->s3) {
throw new \RuntimeException('The S3 storage is unavailable, so remote backup archives cannot be deleted.');
}
deleteBackupsS3($s3Filenames, $this->backup->s3);
}
$this->backup->delete();
$this->backup = null;
$this->dispatch('success', 'Volume backup schedule and archives deleted.');
return true;
} catch (Throwable $exception) {
$this->dispatch('error', 'Could not delete the backup archives: '.$exception->getMessage());
return false;
} finally {
$lock->release();
}
}
public function cleanupFailed(): void
{
$this->authorize('update', $this->resource);
$deletedCount = $this->backup?->executions()
->where('status', 'failed')
->where('pause_recovery_pending', false)
->where('s3_cleanup_pending', false)
->where(fn ($query) => $query
->whereNull('filename')
->orWhere('local_storage_deleted', true))
->delete() ?? 0;
$this->dispatch(
$deletedCount > 0 ? 'success' : 'info',
$deletedCount > 0 ? 'Failed backup entries cleaned up.' : 'No safely removable failed backup entries found.',
);
}
public function cleanupDeleted(): void
{
$this->authorize('update', $this->resource);
$deletedCount = $this->backup?->executions()
->where('local_storage_deleted', true)
->where(fn ($query) => $query
->where('s3_storage_deleted', true)
->orWhereNull('s3_uploaded')
->orWhere('s3_uploaded', false))
->delete() ?? 0;
$this->dispatch(
$deletedCount > 0 ? 'success' : 'info',
$deletedCount > 0 ? "Cleaned up {$deletedCount} deleted backup entries." : 'No deleted backup entries found.',
);
}
public function deleteBackup(int $executionId, string $password, array $selectedActions = []): bool|string
{
$this->authorize('update', $this->resource);
if (! verifyPasswordConfirmation($password, $this)) {
return 'The provided password is incorrect.';
}
$execution = $this->backup?->executions()->whereKey($executionId)->first();
if (! $execution) {
$this->dispatch('error', 'Backup execution not found.');
return false;
}
if ($execution->status === 'running' || $execution->pause_recovery_pending || $execution->s3_cleanup_pending) {
$this->dispatch('error', 'Wait for the backup and recovery operations to finish before deleting it.');
return false;
}
try {
$server = $this->backup->server();
if (! $execution->local_storage_deleted && filled($execution->filename)) {
if (! $server) {
throw new \RuntimeException('The server is unavailable.');
}
deleteBackupsLocally($execution->filename, $server, throwError: true);
}
if ($this->delete_backup_s3 && $execution->s3_uploaded && ! $execution->s3_storage_deleted) {
if (! $this->backup->s3) {
throw new \RuntimeException('The S3 storage is unavailable.');
}
deleteBackupsS3($execution->filename, $this->backup->s3);
}
$execution->delete();
$this->delete_backup_s3 = false;
$this->dispatch('success', 'Backup deleted.');
return true;
} catch (Throwable $exception) {
$this->dispatch('error', 'Failed to delete backup: '.$exception->getMessage());
return false;
}
}
public function render()
{
$executions = $this->backup?->executions()->paginate(10);
return view('livewire.project.shared.storages.volume-backups', [
'executions' => $executions ?? collect(),
'latestExecution' => $this->backup?->executions()->first(),
]);
}
private function validateSettings(): bool
{
$this->validate();
if (! validate_cron_expression($this->frequency)) {
$this->addError('frequency', 'The frequency must be a valid cron or human expression.');
return false;
}
if ($this->saveToS3 && ! $this->hasValidS3Storage()) {
$this->addError('s3StorageId', 'Select a usable S3 storage owned by your team.');
return false;
}
$this->disableLocalBackup = $this->saveToS3 && $this->disableLocalBackup;
return true;
}
private function persistBackup(bool $enabled): ScheduledVolumeBackup
{
return ScheduledVolumeBackup::query()->updateOrCreate(
['local_persistent_volume_id' => $this->storage->id],
[
'team_id' => currentTeam()->id,
'frequency' => $this->frequency,
'enabled' => $enabled,
'save_s3' => $this->saveToS3,
'disable_local_backup' => $this->disableLocalBackup,
'pause_during_backup' => $this->pauseDuringBackup,
's3_storage_id' => $this->saveToS3 ? $this->s3StorageId : null,
'retention_amount_locally' => $this->retentionAmountLocally,
'retention_days_locally' => $this->retentionDaysLocally,
'retention_max_storage_locally' => $this->retentionMaxStorageLocally,
'retention_amount_s3' => $this->retentionAmountS3,
'retention_days_s3' => $this->retentionDaysS3,
'retention_max_storage_s3' => $this->retentionMaxStorageS3,
'timeout' => $this->timeout,
],
);
}
private function hasValidS3Storage(): bool
{
return $this->s3StorageId !== null
&& S3Storage::query()
->whereKey($this->s3StorageId)
->where('team_id', currentTeam()->id)
->where('is_usable', true)
->exists();
}
}
+3 -1
View File
@@ -4,6 +4,7 @@ namespace App\Livewire\Storage;
use App\Models\S3Storage;
use App\Models\ScheduledDatabaseBackup;
use App\Models\ScheduledVolumeBackup;
use Illuminate\Auth\Access\AuthorizationException;
use Illuminate\Foundation\Auth\Access\AuthorizesRequests;
use Livewire\Component;
@@ -30,7 +31,8 @@ class Show extends Component
return $this->redirectRoute('storage.index', navigate: true);
}
$this->currentRoute = request()->route()->getName();
$this->backupCount = ScheduledDatabaseBackup::where('s3_storage_id', $this->storage->id)->count();
$this->backupCount = ScheduledDatabaseBackup::where('s3_storage_id', $this->storage->id)->count()
+ ScheduledVolumeBackup::where('s3_storage_id', $this->storage->id)->count();
}
public function delete()
+6
View File
@@ -3,6 +3,7 @@
namespace App\Models;
use Illuminate\Database\Eloquent\Casts\Attribute;
use Illuminate\Database\Eloquent\Relations\HasMany;
use Symfony\Component\Yaml\Yaml;
class LocalPersistentVolume extends BaseModel
@@ -41,6 +42,11 @@ class LocalPersistentVolume extends BaseModel
return $this->morphTo('resource');
}
public function scheduledBackups(): HasMany
{
return $this->hasMany(ScheduledVolumeBackup::class, 'local_persistent_volume_id');
}
protected function customizeName($value)
{
return str($value)->trim()->value;
+15
View File
@@ -69,6 +69,16 @@ class S3Storage extends BaseModel
'save_s3' => false,
's3_storage_id' => null,
]);
$volumeBackupIds = ScheduledVolumeBackup::where('s3_storage_id', $storage->id)->pluck('id');
ScheduledVolumeBackupExecution::whereIn('scheduled_volume_backup_id', $volumeBackupIds)
->update([
's3_storage_deleted' => true,
's3_cleanup_pending' => false,
]);
ScheduledVolumeBackup::whereKey($volumeBackupIds)->update([
'save_s3' => false,
's3_storage_id' => null,
]);
});
}
@@ -101,6 +111,11 @@ class S3Storage extends BaseModel
return $this->hasMany(ScheduledDatabaseBackup::class, 's3_storage_id');
}
public function scheduledVolumeBackups()
{
return $this->hasMany(ScheduledVolumeBackup::class, 's3_storage_id');
}
public function awsUrl()
{
return "{$this->endpoint}/{$this->bucket}";
+82
View File
@@ -0,0 +1,82 @@
<?php
namespace App\Models;
use Illuminate\Database\Eloquent\Relations\BelongsTo;
use Illuminate\Database\Eloquent\Relations\HasMany;
use Illuminate\Database\Eloquent\Relations\HasOne;
class ScheduledVolumeBackup extends BaseModel
{
protected $fillable = [
'uuid',
'local_persistent_volume_id',
'team_id',
's3_storage_id',
'frequency',
'enabled',
'save_s3',
'disable_local_backup',
'pause_during_backup',
'retention_amount_locally',
'retention_days_locally',
'retention_max_storage_locally',
'retention_amount_s3',
'retention_days_s3',
'retention_max_storage_s3',
'timeout',
];
protected function casts(): array
{
return [
'enabled' => 'boolean',
'save_s3' => 'boolean',
'disable_local_backup' => 'boolean',
'pause_during_backup' => 'boolean',
'retention_amount_locally' => 'integer',
'retention_days_locally' => 'integer',
'retention_max_storage_locally' => 'float',
'retention_amount_s3' => 'integer',
'retention_days_s3' => 'integer',
'retention_max_storage_s3' => 'float',
'timeout' => 'integer',
];
}
public function volume(): BelongsTo
{
return $this->belongsTo(LocalPersistentVolume::class, 'local_persistent_volume_id');
}
public function team(): BelongsTo
{
return $this->belongsTo(Team::class);
}
public function s3(): BelongsTo
{
return $this->belongsTo(S3Storage::class, 's3_storage_id');
}
public function executions(): HasMany
{
return $this->hasMany(ScheduledVolumeBackupExecution::class)->latest();
}
public function latestExecution(): HasOne
{
return $this->hasOne(ScheduledVolumeBackupExecution::class)->latestOfMany();
}
public function server(): ?Server
{
$resource = $this->volume?->resource;
if ($resource instanceof ServiceApplication || $resource instanceof ServiceDatabase) {
return $resource->service?->server?->fresh();
}
return $resource?->destination?->server?->fresh();
}
}
@@ -0,0 +1,43 @@
<?php
namespace App\Models;
use Illuminate\Database\Eloquent\Relations\BelongsTo;
class ScheduledVolumeBackupExecution extends BaseModel
{
protected $fillable = [
'uuid',
'scheduled_volume_backup_id',
'status',
'message',
'size',
'filename',
'pause_container_ids',
'pause_recovery_pending',
's3_cleanup_pending',
'finished_at',
'local_storage_deleted',
's3_storage_deleted',
's3_uploaded',
];
protected function casts(): array
{
return [
'size' => 'integer',
'finished_at' => 'datetime',
'pause_container_ids' => 'array',
'pause_recovery_pending' => 'boolean',
's3_cleanup_pending' => 'boolean',
'local_storage_deleted' => 'boolean',
's3_storage_deleted' => 'boolean',
's3_uploaded' => 'boolean',
];
}
public function scheduledVolumeBackup(): BelongsTo
{
return $this->belongsTo(ScheduledVolumeBackup::class);
}
}
+1
View File
@@ -25,6 +25,7 @@ class Checkbox extends Component
public ?string $helper = null,
public string|bool|null $checked = false,
public string|bool $instantSave = false,
public bool $live = false,
public bool $disabled = false,
public string $defaultClass = 'dark:border-neutral-700 text-coolgray-400 dark:bg-coolgray-100 rounded-sm cursor-pointer dark:disabled:bg-base dark:disabled:cursor-not-allowed focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-coollabs dark:focus-visible:ring-warning focus-visible:ring-offset-2 dark:focus-visible:ring-offset-base',
public ?string $canGate = null,