feat(backups): add scheduled persistent volume backups

app/Livewire/Project/Application/Backup/Create.php

@@ -0,0 +1,130 @@
+<?php
+
+namespace App\Livewire\Project\Application\Backup;
+
+use App\Models\Application;
+use App\Models\S3Storage;
+use App\Models\ScheduledVolumeBackup;
+use Illuminate\Foundation\Auth\Access\AuthorizesRequests;
+use Illuminate\Support\Collection;
+use Livewire\Attributes\Locked;
+use Livewire\Component;
+
+class Create extends Component
+{
+    use AuthorizesRequests;
+
+    #[Locked]
+    public Application $application;
+
+    public ?int $selectedVolumeId = null;
+
+    public ?int $volumeId = null;
+
+    public bool $volumeLocked = false;
+
+    public string $frequency = 'daily';
+
+    public bool $saveToS3 = false;
+
+    public ?int $s3StorageId = null;
+
+    public Collection $volumes;
+
+    public Collection $definedS3s;
+
+    protected function rules(): array
+    {
+        return [
+            'volumeId' => ['required', 'integer'],
+            'frequency' => ['required', 'string'],
+            'saveToS3' => ['required', 'boolean'],
+            's3StorageId' => ['nullable', 'integer'],
+        ];
+    }
+
+    public function mount(): void
+    {
+        $this->authorize('view', $this->application);
+        $this->volumes = $this->application->persistentStorages()->orderBy('name')->get();
+        $this->definedS3s = S3Storage::ownedByCurrentTeam()->where('is_usable', true)->get();
+        $this->s3StorageId = $this->definedS3s->first()?->id;
+        $this->volumeLocked = $this->selectedVolumeId !== null;
+        $this->volumeId = $this->selectedVolumeId ?? $this->volumes->first()?->id;
+        $this->loadSelectedBackup();
+    }
+
+    public function updatedVolumeId(): void
+    {
+        $this->loadSelectedBackup();
+    }
+
+    public function submit(): void
+    {
+        $this->authorize('update', $this->application);
+        $this->validate();
+
+        $volume = $this->application->persistentStorages()->whereKey($this->volumeId)->first();
+        if (! $volume) {
+            $this->addError('volumeId', 'Select a volume owned by this application.');
+
+            return;
+        }
+
+        if (! validate_cron_expression($this->frequency)) {
+            $this->addError('frequency', 'The frequency must be a valid cron or human expression.');
+
+            return;
+        }
+
+        if ($this->saveToS3 && ! $this->validS3StorageExists()) {
+            $this->addError('s3StorageId', 'Select a usable S3 storage owned by your team.');
+
+            return;
+        }
+
+        $backup = ScheduledVolumeBackup::query()->updateOrCreate(
+            ['local_persistent_volume_id' => $volume->id],
+            [
+                'team_id' => currentTeam()->id,
+                'frequency' => $this->frequency,
+                'enabled' => true,
+                'save_s3' => $this->saveToS3,
+                's3_storage_id' => $this->saveToS3 ? $this->s3StorageId : null,
+            ],
+        );
+
+        $this->dispatch('success', $backup->wasRecentlyCreated ? 'Scheduled volume backup created.' : 'Scheduled volume backup updated.');
+        $this->dispatch('refreshVolumeBackups');
+        $this->dispatch('close-modal');
+    }
+
+    public function render()
+    {
+        return view('livewire.project.application.backup.create');
+    }
+
+    private function loadSelectedBackup(): void
+    {
+        $volume = $this->volumes->firstWhere('id', $this->volumeId);
+        if (! $volume) {
+            return;
+        }
+
+        $backup = $volume->scheduledBackups()->first();
+        if ($backup) {
+            $this->frequency = $backup->frequency;
+            $this->saveToS3 = $backup->save_s3;
+            $this->s3StorageId = $backup->s3_storage_id ?? $this->definedS3s->first()?->id;
+        }
+    }
+
+    private function validS3StorageExists(): bool
+    {
+        return $this->s3StorageId !== null
+            && S3Storage::ownedByCurrentTeam()
+                ->where('is_usable', true)
+                ->whereKey($this->s3StorageId)
+                ->exists();
+    }
+}

app/Livewire/Project/Application/Backup/Index.php

@@ -0,0 +1,55 @@
+<?php
+
+namespace App\Livewire\Project\Application\Backup;
+
+use App\Models\Application;
+use App\Models\ScheduledVolumeBackup;
+use Illuminate\Contracts\View\View;
+use Illuminate\Foundation\Auth\Access\AuthorizesRequests;
+use Livewire\Component;
+
+class Index extends Component
+{
+    use AuthorizesRequests;
+
+    public Application $application;
+
+    public array $parameters;
+
+    protected $listeners = ['refreshVolumeBackups' => '$refresh'];
+
+    public function mount(): void
+    {
+        $this->application = $this->findApplication();
+        $this->authorize('view', $this->application);
+        $this->parameters = get_route_parameters();
+    }
+
+    public function render(): View
+    {
+        $volumeIds = $this->application->persistentStorages()->pluck('id');
+        $backups = ScheduledVolumeBackup::query()
+            ->with(['volume', 'latestExecution'])
+            ->withCount('executions')
+            ->whereIn('local_persistent_volume_id', $volumeIds)
+            ->latest()
+            ->get();
+
+        return view('livewire.project.application.backup.index', ['backups' => $backups]);
+    }
+
+    private function findApplication(): Application
+    {
+        $project = currentTeam()->projects()
+            ->where('uuid', request()->route('project_uuid'))
+            ->firstOrFail();
+        $environment = $project->environments()
+            ->where('uuid', request()->route('environment_uuid'))
+            ->firstOrFail();
+
+        return $environment->applications()
+            ->with('destination.server')
+            ->where('uuid', request()->route('application_uuid'))
+            ->firstOrFail();
+    }
+}

app/Livewire/Project/Application/Backup/Show.php

@@ -0,0 +1,46 @@
+<?php
+
+namespace App\Livewire\Project\Application\Backup;
+
+use App\Models\Application;
+use App\Models\ScheduledVolumeBackup;
+use Illuminate\Foundation\Auth\Access\AuthorizesRequests;
+use Livewire\Component;
+
+class Show extends Component
+{
+    use AuthorizesRequests;
+
+    public Application $application;
+
+    public ScheduledVolumeBackup $backup;
+
+    public array $parameters;
+
+    public function mount(): void
+    {
+        $project = currentTeam()->projects()
+            ->where('uuid', request()->route('project_uuid'))
+            ->firstOrFail();
+        $environment = $project->environments()
+            ->where('uuid', request()->route('environment_uuid'))
+            ->firstOrFail();
+        $this->application = $environment->applications()
+            ->with('destination.server')
+            ->where('uuid', request()->route('application_uuid'))
+            ->firstOrFail();
+        $this->authorize('view', $this->application);
+
+        $this->backup = ScheduledVolumeBackup::query()
+            ->with('volume')
+            ->where('uuid', request()->route('backup_uuid'))
+            ->whereIn('local_persistent_volume_id', $this->application->persistentStorages()->pluck('id'))
+            ->firstOrFail();
+        $this->parameters = get_route_parameters();
+    }
+
+    public function render()
+    {
+        return view('livewire.project.application.backup.show');
+    }
+}

app/Livewire/Project/Database/BackupEdit.php

@@ -217,6 +217,19 @@ class BackupEdit extends Component
         }
     }
 
+    public function toggleEnabled(): void
+    {
+        try {
+            $this->authorize('manageBackups', $this->backup->database);
+
+            $this->backupEnabled = ! $this->backupEnabled;
+            $this->backup->update(['enabled' => $this->backupEnabled]);
+            $this->dispatch('success', $this->backupEnabled ? 'Backup enabled.' : 'Backup disabled.');
+        } catch (\Throwable $e) {
+            $this->dispatch('error', $e->getMessage());
+        }
+    }
+
     public function updatedS3StorageId(): void
     {
         $this->instantSave();

app/Livewire/Project/Shared/Storages/Show.php

@@ -5,6 +5,7 @@ namespace App\Livewire\Project\Shared\Storages;
 use App\Models\LocalPersistentVolume;
 use App\Support\ValidationPatterns;
 use Illuminate\Foundation\Auth\Access\AuthorizesRequests;
+use Livewire\Attributes\On;
 use Livewire\Component;
 
 class Show extends Component
@@ -32,6 +33,8 @@ class Show extends Component
 
     public bool $isPreviewSuffixEnabled = true;
 
+    public bool $hasEnabledBackup = false;
+
     protected $validationAttributes = [
         'name' => 'name',
         'mountPath' => 'mount',
@@ -81,10 +84,19 @@ class Show extends Component
         }
     }
 
-    public function mount()
+    public function mount(): void
     {
         $this->syncData(false);
         $this->isReadOnly = $this->storage->shouldBeReadOnlyInUI();
+        $this->refreshBackupStatus();
+    }
+
+    #[On('refreshVolumeBackups')]
+    public function refreshBackupStatus(): void
+    {
+        $this->hasEnabledBackup = $this->storage->scheduledBackups()
+            ->where('enabled', true)
+            ->exists();
     }
 
     public function instantSave(): void
@@ -115,6 +127,12 @@ class Show extends Component
             return 'The provided password is incorrect.';
         }
 
+        if ($this->storage->scheduledBackups()->exists()) {
+            $this->dispatch('error', 'Delete this volume backup schedule and its archives before deleting the volume.');
+
+            return false;
+        }
+
         $this->storage->delete();
         $this->dispatch('refreshStorages');
 

app/Livewire/Project/Shared/Storages/VolumeBackups.php

@@ -0,0 +1,394 @@
+<?php
+
+namespace App\Livewire\Project\Shared\Storages;
+
+use App\Jobs\VolumeBackupJob;
+use App\Models\LocalPersistentVolume;
+use App\Models\S3Storage;
+use App\Models\ScheduledVolumeBackup;
+use Illuminate\Foundation\Auth\Access\AuthorizesRequests;
+use Illuminate\Support\Collection;
+use Illuminate\Support\Facades\Cache;
+use Livewire\Component;
+use Livewire\WithPagination;
+use Throwable;
+
+class VolumeBackups extends Component
+{
+    use AuthorizesRequests;
+    use WithPagination;
+
+    public LocalPersistentVolume $storage;
+
+    public $resource;
+
+    public ?ScheduledVolumeBackup $backup = null;
+
+    public string $frequency = 'daily';
+
+    public bool $enabled = false;
+
+    public bool $saveToS3 = false;
+
+    public bool $disableLocalBackup = false;
+
+    public bool $pauseDuringBackup = false;
+
+    public ?int $s3StorageId = null;
+
+    public int $retentionAmountLocally = 7;
+
+    public int $retentionDaysLocally = 0;
+
+    public float $retentionMaxStorageLocally = 0;
+
+    public int $retentionAmountS3 = 7;
+
+    public int $retentionDaysS3 = 0;
+
+    public float $retentionMaxStorageS3 = 0;
+
+    public string $timezone = '';
+
+    public int $timeout = 3600;
+
+    public bool $delete_backup_s3 = false;
+
+    public Collection $availableS3Storages;
+
+    protected function rules(): array
+    {
+        return [
+            'frequency' => ['required', 'string'],
+            'enabled' => ['required', 'boolean'],
+            'saveToS3' => ['required', 'boolean'],
+            'disableLocalBackup' => ['required', 'boolean'],
+            'pauseDuringBackup' => ['required', 'boolean'],
+            's3StorageId' => ['nullable', 'integer'],
+            'retentionAmountLocally' => ['required', 'integer', 'min:0', 'max:10000'],
+            'retentionDaysLocally' => ['required', 'integer', 'min:0'],
+            'retentionMaxStorageLocally' => ['required', 'numeric', 'min:0'],
+            'retentionAmountS3' => ['required', 'integer', 'min:0', 'max:10000'],
+            'retentionDaysS3' => ['required', 'integer', 'min:0'],
+            'retentionMaxStorageS3' => ['required', 'numeric', 'min:0'],
+            'timeout' => ['required', 'integer', 'min:60', 'max:36000'],
+        ];
+    }
+
+    public function mount(): void
+    {
+        $this->authorize('view', $this->resource);
+        $this->availableS3Storages = S3Storage::ownedByCurrentTeam()
+            ->where('is_usable', true)
+            ->get();
+        $this->backup = $this->storage->scheduledBackups()->first();
+        $server = $this->backup?->server() ?? data_get($this->resource, 'destination.server');
+        $this->timezone = data_get($server, 'settings.server_timezone', 'Instance timezone');
+
+        if ($this->backup) {
+            $this->frequency = $this->backup->frequency;
+            $this->enabled = $this->backup->enabled;
+            $this->saveToS3 = $this->backup->save_s3;
+            $this->disableLocalBackup = $this->backup->disable_local_backup;
+            $this->pauseDuringBackup = $this->backup->pause_during_backup;
+            $this->s3StorageId = $this->backup->s3_storage_id ?? $this->availableS3Storages->first()?->id;
+            $this->retentionAmountLocally = $this->backup->retention_amount_locally;
+            $this->retentionDaysLocally = $this->backup->retention_days_locally;
+            $this->retentionMaxStorageLocally = $this->backup->retention_max_storage_locally;
+            $this->retentionAmountS3 = $this->backup->retention_amount_s3;
+            $this->retentionDaysS3 = $this->backup->retention_days_s3;
+            $this->retentionMaxStorageS3 = $this->backup->retention_max_storage_s3;
+            $this->timeout = $this->backup->timeout;
+        } else {
+            $this->s3StorageId = $this->availableS3Storages->first()?->id;
+        }
+    }
+
+    public function save(): void
+    {
+        $this->authorize('update', $this->resource);
+
+        if (! $this->validateSettings()) {
+            return;
+        }
+
+        $this->backup = $this->persistBackup($this->enabled);
+        $this->dispatch('success', 'Volume backup schedule saved.');
+    }
+
+    public function instantSave(): void
+    {
+        $this->save();
+    }
+
+    public function updatedS3StorageId(): void
+    {
+        if ($this->saveToS3) {
+            $this->save();
+        }
+    }
+
+    public function toggleEnabled(): void
+    {
+        $this->authorize('update', $this->resource);
+
+        if (! $this->backup) {
+            if (! $this->validateSettings()) {
+                return;
+            }
+
+            $this->enabled = true;
+            $this->backup = $this->persistBackup(true);
+        } else {
+            $this->enabled = ! $this->enabled;
+            $this->backup->update(['enabled' => $this->enabled]);
+        }
+
+        $this->dispatch('success', $this->enabled ? 'Volume backups enabled.' : 'Volume backups disabled.');
+    }
+
+    public function backupNow(): void
+    {
+        $this->authorize('update', $this->resource);
+
+        if (! $this->validateSettings()) {
+            return;
+        }
+
+        if (! $this->backup) {
+            $this->enabled = false;
+        }
+
+        $this->backup = $this->persistBackup($this->enabled);
+        VolumeBackupJob::dispatch($this->backup);
+        $this->dispatch('success', 'Volume backup queued.');
+    }
+
+    public function delete(?string $password = null, array $selectedActions = [])
+    {
+        $this->authorize('update', $this->resource);
+
+        if (! $password || ! verifyPasswordConfirmation($password, $this)) {
+            return 'The provided password is incorrect.';
+        }
+
+        if (! $this->backup) {
+            return false;
+        }
+
+        $lock = Cache::lock(VolumeBackupJob::lockKey($this->backup->id), $this->backup->timeout + 300);
+
+        if (! $lock->get()) {
+            $this->dispatch('error', 'Wait for the queued or running volume backup to finish before deleting this schedule.');
+
+            return false;
+        }
+
+        try {
+            if ($this->backup->executions()
+                ->where(fn ($query) => $query
+                    ->where('status', 'running')
+                    ->orWhere('pause_recovery_pending', true)
+                    ->orWhere('s3_cleanup_pending', true))
+                ->exists()) {
+                $this->dispatch('error', 'Wait for the running volume backup and container recovery to finish before deleting this schedule.');
+
+                return false;
+            }
+
+            $localFilenames = $this->backup->executions()
+                ->where('local_storage_deleted', false)
+                ->pluck('filename')
+                ->filter()
+                ->all();
+            $server = $this->backup->server();
+
+            if ($localFilenames !== []) {
+                if (! $server) {
+                    throw new \RuntimeException('The server is unavailable, so local backup archives cannot be deleted.');
+                }
+
+                deleteBackupsLocally($localFilenames, $server, throwError: true);
+            }
+
+            $s3Filenames = $this->backup->executions()
+                ->where('s3_uploaded', true)
+                ->where('s3_storage_deleted', false)
+                ->pluck('filename')
+                ->filter()
+                ->all();
+
+            if ($s3Filenames !== []) {
+                if (! $this->backup->s3) {
+                    throw new \RuntimeException('The S3 storage is unavailable, so remote backup archives cannot be deleted.');
+                }
+
+                deleteBackupsS3($s3Filenames, $this->backup->s3);
+            }
+
+            $this->backup->delete();
+            $this->backup = null;
+            $this->dispatch('success', 'Volume backup schedule and archives deleted.');
+
+            return true;
+        } catch (Throwable $exception) {
+            $this->dispatch('error', 'Could not delete the backup archives: '.$exception->getMessage());
+
+            return false;
+        } finally {
+            $lock->release();
+        }
+    }
+
+    public function cleanupFailed(): void
+    {
+        $this->authorize('update', $this->resource);
+
+        $deletedCount = $this->backup?->executions()
+            ->where('status', 'failed')
+            ->where('pause_recovery_pending', false)
+            ->where('s3_cleanup_pending', false)
+            ->where(fn ($query) => $query
+                ->whereNull('filename')
+                ->orWhere('local_storage_deleted', true))
+            ->delete() ?? 0;
+
+        $this->dispatch(
+            $deletedCount > 0 ? 'success' : 'info',
+            $deletedCount > 0 ? 'Failed backup entries cleaned up.' : 'No safely removable failed backup entries found.',
+        );
+    }
+
+    public function cleanupDeleted(): void
+    {
+        $this->authorize('update', $this->resource);
+
+        $deletedCount = $this->backup?->executions()
+            ->where('local_storage_deleted', true)
+            ->where(fn ($query) => $query
+                ->where('s3_storage_deleted', true)
+                ->orWhereNull('s3_uploaded')
+                ->orWhere('s3_uploaded', false))
+            ->delete() ?? 0;
+
+        $this->dispatch(
+            $deletedCount > 0 ? 'success' : 'info',
+            $deletedCount > 0 ? "Cleaned up {$deletedCount} deleted backup entries." : 'No deleted backup entries found.',
+        );
+    }
+
+    public function deleteBackup(int $executionId, string $password, array $selectedActions = []): bool|string
+    {
+        $this->authorize('update', $this->resource);
+
+        if (! verifyPasswordConfirmation($password, $this)) {
+            return 'The provided password is incorrect.';
+        }
+
+        $execution = $this->backup?->executions()->whereKey($executionId)->first();
+        if (! $execution) {
+            $this->dispatch('error', 'Backup execution not found.');
+
+            return false;
+        }
+
+        if ($execution->status === 'running' || $execution->pause_recovery_pending || $execution->s3_cleanup_pending) {
+            $this->dispatch('error', 'Wait for the backup and recovery operations to finish before deleting it.');
+
+            return false;
+        }
+
+        try {
+            $server = $this->backup->server();
+            if (! $execution->local_storage_deleted && filled($execution->filename)) {
+                if (! $server) {
+                    throw new \RuntimeException('The server is unavailable.');
+                }
+
+                deleteBackupsLocally($execution->filename, $server, throwError: true);
+            }
+
+            if ($this->delete_backup_s3 && $execution->s3_uploaded && ! $execution->s3_storage_deleted) {
+                if (! $this->backup->s3) {
+                    throw new \RuntimeException('The S3 storage is unavailable.');
+                }
+
+                deleteBackupsS3($execution->filename, $this->backup->s3);
+            }
+
+            $execution->delete();
+            $this->delete_backup_s3 = false;
+            $this->dispatch('success', 'Backup deleted.');
+
+            return true;
+        } catch (Throwable $exception) {
+            $this->dispatch('error', 'Failed to delete backup: '.$exception->getMessage());
+
+            return false;
+        }
+    }
+
+    public function render()
+    {
+        $executions = $this->backup?->executions()->paginate(10);
+
+        return view('livewire.project.shared.storages.volume-backups', [
+            'executions' => $executions ?? collect(),
+            'latestExecution' => $this->backup?->executions()->first(),
+        ]);
+    }
+
+    private function validateSettings(): bool
+    {
+        $this->validate();
+
+        if (! validate_cron_expression($this->frequency)) {
+            $this->addError('frequency', 'The frequency must be a valid cron or human expression.');
+
+            return false;
+        }
+
+        if ($this->saveToS3 && ! $this->hasValidS3Storage()) {
+            $this->addError('s3StorageId', 'Select a usable S3 storage owned by your team.');
+
+            return false;
+        }
+
+        $this->disableLocalBackup = $this->saveToS3 && $this->disableLocalBackup;
+
+        return true;
+    }
+
+    private function persistBackup(bool $enabled): ScheduledVolumeBackup
+    {
+        return ScheduledVolumeBackup::query()->updateOrCreate(
+            ['local_persistent_volume_id' => $this->storage->id],
+            [
+                'team_id' => currentTeam()->id,
+                'frequency' => $this->frequency,
+                'enabled' => $enabled,
+                'save_s3' => $this->saveToS3,
+                'disable_local_backup' => $this->disableLocalBackup,
+                'pause_during_backup' => $this->pauseDuringBackup,
+                's3_storage_id' => $this->saveToS3 ? $this->s3StorageId : null,
+                'retention_amount_locally' => $this->retentionAmountLocally,
+                'retention_days_locally' => $this->retentionDaysLocally,
+                'retention_max_storage_locally' => $this->retentionMaxStorageLocally,
+                'retention_amount_s3' => $this->retentionAmountS3,
+                'retention_days_s3' => $this->retentionDaysS3,
+                'retention_max_storage_s3' => $this->retentionMaxStorageS3,
+                'timeout' => $this->timeout,
+            ],
+        );
+    }
+
+    private function hasValidS3Storage(): bool
+    {
+        return $this->s3StorageId !== null
+            && S3Storage::query()
+                ->whereKey($this->s3StorageId)
+                ->where('team_id', currentTeam()->id)
+                ->where('is_usable', true)
+                ->exists();
+    }
+}