Merge remote-tracking branch 'origin/next' into volume-backups-server-s3

app/Actions/Stripe/SyncStripeSubscriptions.php

@@ -1,30 +1,18 @@
 <?php
 
-namespace App\Jobs;
+namespace App\Actions\Stripe;
 
 use App\Models\Subscription;
-use Illuminate\Bus\Queueable;
-use Illuminate\Contracts\Queue\ShouldBeEncrypted;
-use Illuminate\Contracts\Queue\ShouldQueue;
-use Illuminate\Foundation\Bus\Dispatchable;
-use Illuminate\Queue\InteractsWithQueue;
-use Illuminate\Queue\SerializesModels;
+use Lorisleiva\Actions\Concerns\AsAction;
 use Stripe\StripeClient;
 
-class SyncStripeSubscriptionsJob implements ShouldBeEncrypted, ShouldQueue
+class SyncStripeSubscriptions
 {
-    use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
+    use AsAction;
 
-    public int $tries = 1;
+    private const VALID_STRIPE_STATUSES = ['active', 'past_due'];
 
-    public int $timeout = 1800; // 30 minutes max
-
-    public function __construct(public bool $fix = false)
-    {
-        $this->onQueue('high');
-    }
-
-    public function handle(?\Closure $onProgress = null): array
+    public function handle(bool $fix = false, ?\Closure $onProgress = null): array
     {
         if (! isCloud() || ! isStripe()) {
             return ['error' => 'Not running on Cloud or Stripe not configured'];
@@ -34,7 +22,9 @@ class SyncStripeSubscriptionsJob implements ShouldBeEncrypted, ShouldQueue
             ->where('stripe_invoice_paid', true)
             ->get();
 
-        $stripe = app(StripeClient::class);
+        $stripe = app()->bound(StripeClient::class)
+            ? app(StripeClient::class)
+            : new StripeClient(config('subscription.stripe_api_key'));
 
         // Bulk fetch all valid subscription IDs from Stripe (active + past_due)
         $validStripeIds = $this->fetchValidStripeSubscriptionIds($stripe, $onProgress);
@@ -43,13 +33,20 @@ class SyncStripeSubscriptionsJob implements ShouldBeEncrypted, ShouldQueue
         $staleSubscriptions = $subscriptions->filter(
             fn (Subscription $sub) => ! in_array($sub->stripe_subscription_id, $validStripeIds)
         );
+        $staleSubscriptionCount = $staleSubscriptions->count();
+
+        $onProgress?->__invoke('checking', 0, $staleSubscriptionCount);
 
         // For each stale subscription, get the exact Stripe status and check for resubscriptions
         $discrepancies = [];
         $resubscribed = [];
         $errors = [];
+        $fixedCount = 0;
+        $manualReviewCount = 0;
+
+        foreach ($staleSubscriptions->values() as $index => $subscription) {
+            $onProgress?->__invoke('checking', $index + 1, $staleSubscriptionCount);
 
-        foreach ($staleSubscriptions as $subscription) {
             try {
                 $stripeSubscription = $stripe->subscriptions->retrieve(
                     $subscription->stripe_subscription_id
@@ -66,8 +63,18 @@ class SyncStripeSubscriptionsJob implements ShouldBeEncrypted, ShouldQueue
                 continue;
             }
 
-            // Check if this user resubscribed under a different customer/subscription
+            if (in_array($stripeStatus, self::VALID_STRIPE_STATUSES, true)) {
+                continue;
+            }
+
             $activeSub = $this->findActiveSubscriptionByEmail($stripe, $stripeSubscription->customer);
+            $validReplacement = Subscription::query()
+                ->where('team_id', $subscription->team_id)
+                ->where('id', '!=', $subscription->id)
+                ->where('stripe_invoice_paid', true)
+                ->whereIn('stripe_subscription_id', $validStripeIds)
+                ->first();
+
             if ($activeSub) {
                 $resubscribed[] = [
                     'subscription_id' => $subscription->id,
@@ -78,33 +85,69 @@ class SyncStripeSubscriptionsJob implements ShouldBeEncrypted, ShouldQueue
                     'new_stripe_subscription_id' => $activeSub['subscription_id'],
                     'new_stripe_customer_id' => $activeSub['customer_id'],
                     'new_status' => $activeSub['status'],
+                    'linked_to_team' => $validReplacement?->stripe_subscription_id === $activeSub['subscription_id'],
                 ];
-
-                continue;
             }
 
+            $inactiveSubscription = null;
+            if (! $validReplacement && ! $activeSub) {
+                $inactiveSubscription = Subscription::query()
+                    ->where('team_id', $subscription->team_id)
+                    ->where('id', '!=', $subscription->id)
+                    ->where('stripe_invoice_paid', false)
+                    ->first();
+            }
+
+            $resolution = match (true) {
+                (bool) $validReplacement => 'delete_stale',
+                (bool) $activeSub => 'manual_review',
+                (bool) $inactiveSubscription => 'delete_stale',
+                default => 'end_subscription',
+            };
+
             $discrepancies[] = [
                 'subscription_id' => $subscription->id,
                 'team_id' => $subscription->team_id,
                 'stripe_subscription_id' => $subscription->stripe_subscription_id,
                 'stripe_status' => $stripeStatus,
+                'resolution' => $resolution,
             ];
 
-            if ($this->fix) {
-                $subscription->update([
-                    'stripe_invoice_paid' => false,
-                    'stripe_past_due' => false,
-                ]);
+            if ($fix) {
+                $team = $subscription->team;
 
-                if ($stripeStatus === 'canceled') {
-                    $subscription->team?->subscriptionEnded();
+                if ($resolution === 'manual_review') {
+                    $manualReviewCount++;
+
+                    continue;
                 }
+
+                if ($resolution === 'delete_stale') {
+                    if (! $validReplacement && $inactiveSubscription && $team) {
+                        $team->subscriptionEnded($inactiveSubscription);
+                    }
+
+                    $subscription->delete();
+                    $fixedCount++;
+
+                    continue;
+                }
+
+                if ($team) {
+                    $team->subscriptionEnded($subscription);
+                } else {
+                    $subscription->update([
+                        'stripe_invoice_paid' => false,
+                        'stripe_past_due' => false,
+                    ]);
+                }
+                $fixedCount++;
             }
         }
 
-        if ($this->fix && count($discrepancies) > 0) {
+        if ($fix && $fixedCount > 0) {
             send_internal_notification(
-                'SyncStripeSubscriptionsJob: Fixed '.count($discrepancies)." discrepancies:\n".
+                "SyncStripeSubscriptions: Fixed {$fixedCount} discrepancies:\n".
                 json_encode($discrepancies, JSON_PRETTY_PRINT)
             );
         }
@@ -114,7 +157,9 @@ class SyncStripeSubscriptionsJob implements ShouldBeEncrypted, ShouldQueue
             'discrepancies' => $discrepancies,
             'resubscribed' => $resubscribed,
             'errors' => $errors,
-            'fixed' => $this->fix,
+            'fixed' => $fix,
+            'fixed_count' => $fixedCount,
+            'manual_review_count' => $manualReviewCount,
         ];
     }
 
@@ -183,13 +228,13 @@ class SyncStripeSubscriptionsJob implements ShouldBeEncrypted, ShouldQueue
         $validIds = [];
         $fetched = 0;
 
-        foreach (['active', 'past_due'] as $status) {
+        foreach (self::VALID_STRIPE_STATUSES as $status) {
             foreach ($stripe->subscriptions->all(['status' => $status, 'limit' => 100])->autoPagingIterator() as $sub) {
                 $validIds[] = $sub->id;
                 $fetched++;
 
                 if ($onProgress) {
-                    $onProgress($fetched);
+                    $onProgress('fetching', $fetched, null);
                 }
             }
         }

app/Console/Commands/Cloud/CleanupUnverifiedUsers.php

@@ -0,0 +1,83 @@
+<?php
+
+namespace App\Console\Commands\Cloud;
+
+use App\Models\User;
+use Illuminate\Console\Command;
+use Illuminate\Database\Eloquent\Builder;
+use Illuminate\Support\Str;
+
+class CleanupUnverifiedUsers extends Command
+{
+    protected $signature = 'cloud:cleanup-unverified-users
+                            {--yes : Delete eligible users instead of running a dry run}';
+
+    protected $description = 'Delete unverified users without Stripe subscriptions or defined resources';
+
+    public function handle(): int
+    {
+        if (! isCloud()) {
+            $this->error('This command can only be run on Coolify Cloud.');
+
+            return self::FAILURE;
+        }
+
+        $eligibleUsers = $this->eligibleUsers();
+        $eligibleCount = $eligibleUsers->count();
+
+        $this->info("Found {$eligibleCount} ".Str::plural('unverified user', $eligibleCount).' eligible for deletion.');
+        $shouldDelete = (bool) $this->option('yes');
+
+        if (! $shouldDelete) {
+            $this->warn('Dry run only. Use --yes to delete eligible users.');
+        }
+
+        $deletedCount = 0;
+
+        if ($eligibleCount > 0) {
+            $progressAction = $shouldDelete ? 'Deleting' : 'Checking';
+            $progressBar = $this->output->createProgressBar($eligibleCount);
+            $progressBar->setFormat("{$progressAction} eligible users: %current%/%max% [%bar%] %percent:3s%%");
+            $progressBar->start();
+
+            foreach ($eligibleUsers->lazyById(100) as $user) {
+                if ($shouldDelete && $user->delete()) {
+                    $deletedCount++;
+                }
+
+                $progressBar->advance();
+            }
+
+            $progressBar->finish();
+            $this->newLine(2);
+        }
+
+        if ($shouldDelete) {
+            $this->info("Deleted {$deletedCount} ".Str::plural('unverified user', $deletedCount).'.');
+        }
+
+        return self::SUCCESS;
+    }
+
+    private function eligibleUsers(): Builder
+    {
+        return User::query()
+            ->where('id', '!=', 0)
+            ->whereNull('email_verified_at')
+            ->whereDoesntHave('teams', fn (Builder $query) => $query->whereKey(0))
+            ->whereDoesntHave('teams.subscription')
+            ->whereDoesntHave('teams.servers')
+            ->whereDoesntHave('teams', function (Builder $query) {
+                $query->whereHas('projects.applications')
+                    ->orWhereHas('projects.postgresqls')
+                    ->orWhereHas('projects.redis')
+                    ->orWhereHas('projects.mongodbs')
+                    ->orWhereHas('projects.mysqls')
+                    ->orWhereHas('projects.mariadbs')
+                    ->orWhereHas('projects.keydbs')
+                    ->orWhereHas('projects.dragonflies')
+                    ->orWhereHas('projects.clickhouses')
+                    ->orWhereHas('projects.services');
+            });
+    }
+}

app/Console/Commands/Cloud/ExportUsers.php

@@ -0,0 +1,127 @@
+<?php
+
+namespace App\Console\Commands\Cloud;
+
+use App\Models\User;
+use Illuminate\Console\Command;
+use Illuminate\Support\Facades\Storage;
+use RuntimeException;
+use Throwable;
+
+class ExportUsers extends Command
+{
+    protected $signature = 'cloud:export-users';
+
+    protected $description = 'Export subscribed and unsubscribed verified Coolify Cloud users to separate CSV files';
+
+    public function handle(): int
+    {
+        if (! isCloud()) {
+            $this->error('This command can only be run on Coolify Cloud.');
+
+            return self::FAILURE;
+        }
+
+        $backups = Storage::disk('backups');
+        $backups->delete('cloud-users.csv');
+
+        $subscribedPath = $backups->path('cloud-users-subscribed.csv');
+        $unsubscribedPath = $backups->path('cloud-users-unsubscribed.csv');
+        $subscribedOutput = fopen($subscribedPath, 'wb');
+
+        if ($subscribedOutput === false) {
+            $this->error("Unable to open {$subscribedPath} for writing.");
+
+            return self::FAILURE;
+        }
+
+        $unsubscribedOutput = fopen($unsubscribedPath, 'wb');
+
+        if ($unsubscribedOutput === false) {
+            fclose($subscribedOutput);
+            $this->error("Unable to open {$unsubscribedPath} for writing.");
+
+            return self::FAILURE;
+        }
+
+        $subscribedCount = 0;
+        $unsubscribedCount = 0;
+
+        try {
+            $header = [
+                'email',
+                'first_name',
+                'last_name',
+                'lifetime_value_currency',
+                'lifetime_value_amount',
+                'utm_campaign',
+                'utm_source',
+                'utm_medium',
+                'utm_content',
+                'utm_term',
+                'phone',
+            ];
+
+            $this->writeCsvRow($subscribedOutput, $header);
+            $this->writeCsvRow($unsubscribedOutput, $header);
+
+            foreach (User::query()
+                ->select(['id', 'email', 'name'])
+                ->where('id', '!=', 0)
+                ->whereNotNull('email_verified_at')
+                ->withExists([
+                    'teams as is_subscribed' => fn ($query) => $query
+                        ->whereRelation('subscription', 'stripe_invoice_paid', true),
+                ])
+                ->lazyById(500) as $user) {
+                $nameParts = preg_split('/\s+/u', trim((string) $user->name), 2) ?: [];
+                [$firstName, $lastName] = array_pad($nameParts, 2, '');
+
+                $row = [
+                    $user->email,
+                    $firstName,
+                    $lastName,
+                    '',
+                    '',
+                    '',
+                    '',
+                    '',
+                    '',
+                    '',
+                    '',
+                ];
+
+                if ($user->is_subscribed) {
+                    $this->writeCsvRow($subscribedOutput, $row);
+                    $subscribedCount++;
+                } else {
+                    $this->writeCsvRow($unsubscribedOutput, $row);
+                    $unsubscribedCount++;
+                }
+            }
+        } catch (Throwable $exception) {
+            $this->error("Unable to export users: {$exception->getMessage()}");
+
+            return self::FAILURE;
+        } finally {
+            fclose($subscribedOutput);
+            fclose($unsubscribedOutput);
+        }
+
+        $this->info("Exported {$subscribedCount} subscribed verified users to {$subscribedPath}");
+        $this->info("Exported {$unsubscribedCount} unsubscribed verified users to {$unsubscribedPath}");
+
+        return self::SUCCESS;
+    }
+
+    /**
+     * @param  resource  $output
+     * @param  array<int, mixed>  $fields
+     */
+    private function writeCsvRow($output, array $fields): void
+    {
+        if (fputcsv($output, $fields, ',', '"', '') === false) {
+            throw new RuntimeException('Unable to write the CSV file.');
+        }
+    }
+}

app/Console/Commands/Cloud/SyncStripeSubscriptions.php

@@ -2,7 +2,7 @@
 
 namespace App\Console\Commands\Cloud;
 
-use App\Jobs\SyncStripeSubscriptionsJob;
+use App\Actions\Stripe\SyncStripeSubscriptions as SyncStripeSubscriptionsAction;
 use Illuminate\Console\Command;
 
 class SyncStripeSubscriptions extends Command
@@ -35,14 +35,18 @@ class SyncStripeSubscriptions extends Command
 
         $this->newLine();
 
-        $job = new SyncStripeSubscriptionsJob($fix);
-        $fetched = 0;
-        $result = $job->handle(function (int $count) use (&$fetched): void {
-            $fetched = $count;
-            $this->output->write("\r  Fetching subscriptions from Stripe... {$fetched}");
+        $progressShown = false;
+        $result = SyncStripeSubscriptionsAction::run($fix, function (string $stage, int $current, ?int $total) use (&$progressShown): void {
+            $progressShown = true;
+            $message = match ($stage) {
+                'checking' => "  Checking stale subscriptions against Stripe... {$current}/{$total}",
+                default => "  Fetching valid subscriptions from Stripe... {$current}",
+            };
+
+            $this->output->write("\r".str_pad($message, 80));
         });
-        if ($fetched > 0) {
-            $this->output->write("\r".str_repeat(' ', 60)."\r");
+        if ($progressShown) {
+            $this->output->write("\r".str_repeat(' ', 80)."\r");
         }
 
         if (isset($result['error'])) {
@@ -63,13 +67,22 @@ class SyncStripeSubscriptions extends Command
                 $this->line("    Team ID: {$discrepancy['team_id']}");
                 $this->line("    Stripe ID: {$discrepancy['stripe_subscription_id']}");
                 $this->line("    Stripe Status: {$discrepancy['stripe_status']}");
+                $resolution = match ($discrepancy['resolution']) {
+                    'delete_stale' => 'Delete stale local row',
+                    'manual_review' => 'Manual review required',
+                    default => 'End local subscription',
+                };
+                $this->line("    Resolution: {$resolution}");
                 $this->newLine();
             }
 
             if ($fix) {
-                $this->info('All discrepancies have been fixed.');
+                $this->info("Automatic corrections applied: {$result['fixed_count']}");
+                if ($result['manual_review_count'] > 0) {
+                    $this->warn("Skipped for manual review: {$result['manual_review_count']}");
+                }
             } else {
-                $this->comment('Run with --fix to correct these discrepancies.');
+                $this->comment('Run with --fix to apply automatic corrections.');
             }
         } else {
             $this->info('No discrepancies found. All subscriptions are in sync.');
@@ -84,6 +97,7 @@ class SyncStripeSubscriptions extends Command
                 $this->line("  - Team ID: {$resub['team_id']} | Email: {$resub['email']}");
                 $this->line("    Old: {$resub['old_stripe_subscription_id']} (cus: {$resub['old_stripe_customer_id']})");
                 $this->line("    New: {$resub['new_stripe_subscription_id']} (cus: {$resub['new_stripe_customer_id']}) [{$resub['new_status']}]");
+                $this->line('    Linked to this team: '.($resub['linked_to_team'] ? 'Yes' : 'No'));
                 $this->newLine();
             }
         }

app/Http/Controllers/Api/DatabasesController.php

@@ -850,6 +850,12 @@ class DatabasesController extends Controller
 
         $this->authorize('manageBackups', $database);
 
+        if (! $database->isBackupSolutionAvailable()) {
+            return response()->json([
+                'message' => 'Scheduled backups are not supported for this database type.',
+            ], 422);
+        }
+
         // Validate frequency is a valid cron expression
         $isValid = validate_cron_expression($request->frequency);
         if (! $isValid) {
@@ -915,6 +921,8 @@ class DatabasesController extends Controller
                 $backupData['databases_to_backup'] = $database->mysql_database;
             } elseif ($database->type() === 'standalone-mariadb') {
                 $backupData['databases_to_backup'] = $database->mariadb_database;
+            } elseif ($database->type() === 'standalone-clickhouse') {
+                $backupData['databases_to_backup'] = $database->clickhouse_db;
             }
         }
 
@@ -3016,7 +3024,7 @@ class DatabasesController extends Controller
             ),
         ]
     )]
-    public function move_by_uuid(Request $request): \Illuminate\Http\JsonResponse
+    public function move_by_uuid(Request $request): JsonResponse
     {
         $teamId = getTeamIdFromToken();
         if (is_null($teamId)) {

app/Http/Controllers/Api/ServiceApplicationsController.php

@@ -490,7 +490,7 @@ class ServiceApplicationsController extends Controller
             ], 400);
         }
 
-        $lines = (int) ($request->query('lines', 100) ?: 100);
+        $lines = normalizeLogLines($request->query('lines'));
         $logs = getContainerLogs($server, $containerName, $lines);
 
         return response()->json([

app/Http/Controllers/Api/ServiceDatabasesController.php

@@ -311,7 +311,7 @@ class ServiceDatabasesController extends Controller
             return response()->json(['message' => 'Service database container is not running.'], 400);
         }
 
-        $lines = (int) ($request->query('lines', 100) ?: 100);
+        $lines = normalizeLogLines($request->query('lines'));
 
         return response()->json([
             'logs' => getContainerLogs($server, $containerName, $lines),

app/Http/Controllers/Api/VultrController.php

@@ -15,6 +15,7 @@ use App\Rules\ValidHostname;
 use App\Services\VultrService;
 use Illuminate\Http\JsonResponse;
 use Illuminate\Http\Request;
+use Illuminate\Support\Facades\DB;
 use OpenApi\Attributes as OA;
 
 class VultrController extends Controller
@@ -286,6 +287,10 @@ class VultrController extends Controller
             return response()->json(['message' => 'Private key not found.'], 404);
         }
 
+        $vultrService = null;
+        $vultrInstanceId = null;
+        $server = null;
+
         try {
             $vultrService = new VultrService($token->token);
             $publicKey = $privateKey->getPublicKey();
@@ -317,33 +322,41 @@ class VultrController extends Controller
             }
 
             $vultrInstance = $vultrService->createInstance($params);
+            $vultrInstanceId = (string) $vultrInstance['id'];
             $ipAddress = $vultrService->getPublicIp($vultrInstance, $request->disable_public_ipv4, $request->enable_ipv6) ?? Server::PLACEHOLDER_IP;
 
-            $server = Server::create([
-                'name' => $normalizedServerName,
-                'ip' => $ipAddress,
-                'user' => 'root',
-                'port' => 22,
-                'team_id' => $teamId,
-                'private_key_id' => $privateKey->id,
-                'cloud_provider_token_id' => $token->id,
-                'vultr_instance_id' => $vultrInstance['id'],
-                'vultr_instance_status' => $vultrInstance['status'] ?? null,
-            ]);
-
-            $vultrInstance = $vultrService->waitForPublicIp($vultrInstance, $request->disable_public_ipv4, $request->enable_ipv6);
-            $assignedIpAddress = $vultrService->getPublicIp($vultrInstance, $request->disable_public_ipv4, $request->enable_ipv6);
-            if ($assignedIpAddress && $assignedIpAddress !== $server->ip) {
-                $ipAddress = $assignedIpAddress;
-                $server->update([
-                    'ip' => $assignedIpAddress,
-                    'vultr_instance_status' => $vultrInstance['status'] ?? $server->vultr_instance_status,
+            $server = DB::transaction(function () use ($normalizedServerName, $ipAddress, $teamId, $privateKey, $token, $vultrInstanceId, $vultrInstance): Server {
+                $server = Server::create([
+                    'name' => $normalizedServerName,
+                    'ip' => $ipAddress,
+                    'user' => 'root',
+                    'port' => 22,
+                    'team_id' => $teamId,
+                    'private_key_id' => $privateKey->id,
+                    'cloud_provider_token_id' => $token->id,
+                    'vultr_instance_id' => $vultrInstanceId,
+                    'vultr_instance_status' => $vultrInstance['status'] ?? null,
                 ]);
-            }
 
-            $server->proxy->set('status', 'exited');
-            $server->proxy->set('type', ProxyTypes::TRAEFIK->value);
-            $server->save();
+                $server->proxy->set('status', 'exited');
+                $server->proxy->set('type', ProxyTypes::TRAEFIK->value);
+                $server->save();
+
+                return $server;
+            });
+
+            try {
+                $vultrInstance = $vultrService->waitForPublicIp($vultrInstance, $request->disable_public_ipv4, $request->enable_ipv6);
+                $assignedIpAddress = $vultrService->getPublicIp($vultrInstance, $request->disable_public_ipv4, $request->enable_ipv6);
+                if ($assignedIpAddress && $assignedIpAddress !== $server->ip) {
+                    $server->update([
+                        'ip' => $assignedIpAddress,
+                        'vultr_instance_status' => $vultrInstance['status'] ?? $server->vultr_instance_status,
+                    ]);
+                }
+            } catch (\Throwable $e) {
+                report($e);
+            }
 
             if ($request->instant_validate) {
                 ValidateServer::dispatch($server);
@@ -353,27 +366,48 @@ class VultrController extends Controller
                 'team_id' => $teamId,
                 'server_uuid' => $server->uuid,
                 'server_name' => $server->name,
-                'vultr_instance_id' => $vultrInstance['id'],
-                'ip' => $ipAddress,
+                'vultr_instance_id' => $vultrInstanceId,
+                'ip' => $server->ip,
             ]);
 
             return response()->json([
                 'uuid' => $server->uuid,
-                'vultr_instance_id' => $vultrInstance['id'],
-                'ip' => $ipAddress,
+                'vultr_instance_id' => $vultrInstanceId,
+                'ip' => $server->ip,
             ])->setStatusCode(201);
         } catch (RateLimitException $e) {
+            $this->deleteUntrackedInstance($vultrService, $vultrInstanceId, $server);
+
             $response = response()->json(['message' => $e->getMessage()], 429);
             if ($e->retryAfter !== null) {
                 $response->header('Retry-After', $e->retryAfter);
             }
 
             return $response;
-        } catch (\Throwable) {
+        } catch (\Throwable $e) {
+            $this->deleteUntrackedInstance($vultrService, $vultrInstanceId, $server);
+
+            logger()->error('Failed to create Vultr server', [
+                'error' => $e->getMessage(),
+            ]);
+
             return response()->json(['message' => 'Failed to create Vultr server.'], 500);
         }
     }
 
+    private function deleteUntrackedInstance(?VultrService $vultrService, ?string $vultrInstanceId, ?Server $server): void
+    {
+        if (! $vultrService || ! $vultrInstanceId || $server) {
+            return;
+        }
+
+        try {
+            $vultrService->deleteInstance($vultrInstanceId);
+        } catch (\Throwable $e) {
+            report($e);
+        }
+    }
+
     private function findMatchingSshKey(array $sshKeys, string $publicKey): ?array
     {
         $normalizedPublicKey = $this->normalizePublicKey($publicKey);

app/Jobs/DatabaseBackupJob.php

@@ -8,6 +8,7 @@ use App\Models\ScheduledDatabaseBackup;
 use App\Models\ScheduledDatabaseBackupExecution;
 use App\Models\Server;
 use App\Models\ServiceDatabase;
+use App\Models\StandaloneClickhouse;
 use App\Models\StandaloneMariadb;
 use App\Models\StandaloneMongodb;
 use App\Models\StandaloneMysql;
@@ -17,6 +18,7 @@ use App\Notifications\Database\BackupFailed;
 use App\Notifications\Database\BackupSuccess;
 use App\Notifications\Database\BackupSuccessWithS3Warning;
 use App\Rules\SafeWebhookUrl;
+use App\Support\ClickhouseBackupCommand;
 use Carbon\Carbon;
 use Illuminate\Bus\Queueable;
 use Illuminate\Contracts\Queue\ShouldBeEncrypted;
@@ -39,7 +41,7 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue
 
     public Server $server;
 
-    public StandalonePostgresql|StandaloneMongodb|StandaloneMysql|StandaloneMariadb|ServiceDatabase $database;
+    public StandalonePostgresql|StandaloneMongodb|StandaloneMysql|StandaloneMariadb|StandaloneClickhouse|ServiceDatabase $database;
 
     public ?string $container_name = null;
 
@@ -271,6 +273,8 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue
                     $databasesToBackup = [$this->database->mysql_database];
                 } elseif (str($databaseType)->contains('mariadb')) {
                     $databasesToBackup = [$this->database->mariadb_database];
+                } elseif ($this->database instanceof StandaloneClickhouse) {
+                    $databasesToBackup = [$this->database->clickhouse_db];
                 } else {
                     return;
                 }
@@ -294,6 +298,10 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue
                     // Format: db1,db2,db3
                     $databasesToBackup = explode(',', $databasesToBackup);
                     $databasesToBackup = array_map('trim', $databasesToBackup);
+                } elseif ($this->database instanceof StandaloneClickhouse) {
+                    // Format: db1,db2,db3
+                    $databasesToBackup = explode(',', $databasesToBackup);
+                    $databasesToBackup = array_map('trim', $databasesToBackup);
                 } else {
                     return;
                 }
@@ -386,6 +394,17 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue
                             'local_storage_deleted' => false,
                         ]);
                         $this->backup_standalone_mariadb($database);
+                    } elseif ($this->database instanceof StandaloneClickhouse) {
+                        $this->backup_file = '/clickhouse-backup-'.Carbon::now()->timestamp."-{$this->backup_log_uuid}.zip";
+                        $this->backup_location = $this->backup_dir.$this->backup_file;
+                        $this->backup_log = ScheduledDatabaseBackupExecution::create([
+                            'uuid' => $this->backup_log_uuid,
+                            'database_name' => $database,
+                            'filename' => $this->backup_location,
+                            'scheduled_database_backup_id' => $this->backup->id,
+                            'local_storage_deleted' => false,
+                        ]);
+                        $this->backup_standalone_clickhouse($database);
                     } else {
                         throw new \Exception('Unsupported database type');
                     }
@@ -400,6 +419,9 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue
                     }
                 } catch (Throwable $e) {
                     // Local backup failed
+                    if ($this->database instanceof StandaloneClickhouse) {
+                        deleteBackupsLocally($this->backup_location, $this->server);
+                    }
                     if ($this->backup_log) {
                         $this->backup_log->update([
                             'status' => 'failed',
@@ -642,6 +664,32 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue
         }
     }
 
+    private function backup_standalone_clickhouse(string $database): void
+    {
+        $archiveName = ltrim($this->backup_file, '/');
+
+        try {
+            $commands = ClickhouseBackupCommand::make(
+                containerName: $this->container_name,
+                database: $database,
+                archiveName: $archiveName,
+                backupDirectory: $this->backup_dir,
+            );
+
+            $this->backup_output = instant_remote_process($commands, $this->server, true, false, $this->timeout, disableMultiplexing: true);
+            $this->backup_output = trim($this->backup_output);
+            if ($this->backup_output === '') {
+                $this->backup_output = null;
+            }
+        } catch (Throwable $e) {
+            $this->add_to_error_output($e->getMessage());
+            throw $e;
+        } finally {
+            $cleanupCommand = ClickhouseBackupCommand::cleanup($this->container_name, $archiveName);
+            instant_remote_process([$cleanupCommand], $this->server, false, false, null, disableMultiplexing: true);
+        }
+    }
+
     private function add_to_backup_output($output): void
     {
         if ($this->backup_output) {

app/Livewire/Project/Database/Backup/Index.php

@@ -22,13 +22,7 @@ class Index extends Component
         if (! $database) {
             return redirect()->route('dashboard');
         }
-        // No backups
-        if (
-            $database->getMorphClass() === \App\Models\StandaloneRedis::class ||
-            $database->getMorphClass() === \App\Models\StandaloneKeydb::class ||
-            $database->getMorphClass() === \App\Models\StandaloneDragonfly::class ||
-            $database->getMorphClass() === \App\Models\StandaloneClickhouse::class
-        ) {
+        if (! $database->isBackupSolutionAvailable()) {
             return redirect()->route('project.database.configuration', [
                 'project_uuid' => $project->uuid,
                 'environment_uuid' => $environment->uuid,

app/Livewire/Project/Database/CreateScheduledBackup.php

@@ -26,6 +26,12 @@ class CreateScheduledBackup extends Component
         try {
             $this->authorize('manageBackups', $this->database);
 
+            if (! $this->database->isBackupSolutionAvailable()) {
+                $this->dispatch('error', 'Scheduled backups are not supported for this database type.');
+
+                return;
+            }
+
             $this->validate();
 
             $isValid = validate_cron_expression($this->frequency);
@@ -51,6 +57,8 @@ class CreateScheduledBackup extends Component
                 $payload['databases_to_backup'] = $this->database->mysql_database;
             } elseif ($this->database->type() === 'standalone-mariadb') {
                 $payload['databases_to_backup'] = $this->database->mariadb_database;
+            } elseif ($this->database->type() === 'standalone-clickhouse') {
+                $payload['databases_to_backup'] = $this->database->clickhouse_db;
             }
 
             $databaseBackup = ScheduledDatabaseBackup::create($payload);

app/Livewire/Server/New/ByVultr.php

@@ -14,6 +14,7 @@ use App\Services\VultrService;
 use Illuminate\Foundation\Auth\Access\AuthorizesRequests;
 use Illuminate\Http\Client\RequestException;
 use Illuminate\Support\Collection;
+use Illuminate\Support\Facades\DB;
 use Livewire\Attributes\Locked;
 use Livewire\Component;
 
@@ -377,9 +378,8 @@ class ByVultr extends Component
         return "{$providerName} API error: {$details}";
     }
 
-    private function createVultrServer(string $token): array
+    private function createVultrServer(VultrService $vultrService): array
     {
-        $vultrService = new VultrService($token);
         $privateKey = PrivateKey::ownedByCurrentTeam()->findOrFail($this->private_key_id);
         $publicKey = $privateKey->getPublicKey();
         $existingKey = $this->findMatchingSshKey($vultrService->getSshKeys(), $publicKey);
@@ -419,6 +419,10 @@ class ByVultr extends Component
             return null;
         }
 
+        $vultrService = null;
+        $vultrInstanceId = null;
+        $server = null;
+
         try {
             $this->authorize('create', Server::class);
 
@@ -437,20 +441,29 @@ class ByVultr extends Component
             }
 
             $vultrService = new VultrService($this->getVultrToken());
-            $vultrInstance = $this->createVultrServer($this->getVultrToken());
+            $vultrInstance = $this->createVultrServer($vultrService);
+            $vultrInstanceId = (string) $vultrInstance['id'];
             $ipAddress = $vultrService->getPublicIp($vultrInstance, $this->disable_public_ipv4, $this->enable_ipv6) ?? Server::PLACEHOLDER_IP;
 
-            $server = Server::create([
-                'name' => strtolower(trim($this->server_name)),
-                'ip' => $ipAddress,
-                'user' => 'root',
-                'port' => 22,
-                'team_id' => currentTeam()->id,
-                'private_key_id' => $this->private_key_id,
-                'cloud_provider_token_id' => $this->selected_token_id,
-                'vultr_instance_id' => $vultrInstance['id'],
-                'vultr_instance_status' => $vultrInstance['status'] ?? null,
-            ]);
+            $server = DB::transaction(function () use ($ipAddress, $vultrInstanceId, $vultrInstance): Server {
+                $server = Server::create([
+                    'name' => strtolower(trim($this->server_name)),
+                    'ip' => $ipAddress,
+                    'user' => 'root',
+                    'port' => 22,
+                    'team_id' => currentTeam()->id,
+                    'private_key_id' => $this->private_key_id,
+                    'cloud_provider_token_id' => $this->selected_token_id,
+                    'vultr_instance_id' => $vultrInstanceId,
+                    'vultr_instance_status' => $vultrInstance['status'] ?? null,
+                ]);
+
+                $server->proxy->set('status', 'exited');
+                $server->proxy->set('type', ProxyTypes::TRAEFIK->value);
+                $server->save();
+
+                return $server;
+            });
 
             try {
                 $vultrInstance = $vultrService->waitForPublicIp($vultrInstance, $this->disable_public_ipv4, $this->enable_ipv6);
@@ -466,10 +479,6 @@ class ByVultr extends Component
                 report($e);
             }
 
-            $server->proxy->set('status', 'exited');
-            $server->proxy->set('type', ProxyTypes::TRAEFIK->value);
-            $server->save();
-
             if ($this->from_onboarding) {
                 currentTeam()->update([
                     'show_boarding' => false,
@@ -479,10 +488,25 @@ class ByVultr extends Component
 
             return redirectRoute($this, 'server.show', [$server->uuid]);
         } catch (\Throwable $e) {
+            $this->deleteUntrackedInstance($vultrService, $vultrInstanceId, $server);
+
             return handleError($e, $this);
         }
     }
 
+    private function deleteUntrackedInstance(?VultrService $vultrService, ?string $vultrInstanceId, ?Server $server): void
+    {
+        if (! $vultrService || ! $vultrInstanceId || $server) {
+            return;
+        }
+
+        try {
+            $vultrService->deleteInstance($vultrInstanceId);
+        } catch (\Throwable $e) {
+            report($e);
+        }
+    }
+
     public function render()
     {
         return view('livewire.server.new.by-vultr');

app/Models/StandaloneClickhouse.php

@@ -370,6 +370,6 @@ class StandaloneClickhouse extends BaseModel
 
     public function isBackupSolutionAvailable()
     {
-        return false;
+        return true;
     }
 }

app/Models/Team.php

@@ -219,13 +219,15 @@ class Team extends Model implements SendsDiscord, SendsEmail, SendsPushover, Sen
             $this->getNotificationSettings('webhook')?->isEnabled();
     }
 
-    public function subscriptionEnded()
+    public function subscriptionEnded(?Subscription $subscription = null): void
     {
-        if (! $this->subscription) {
+        $subscription ??= $this->subscription;
+
+        if (! $subscription) {
             return;
         }
 
-        $this->subscription->update([
+        $subscription->update([
             'stripe_subscription_id' => null,
             'stripe_cancel_at_period_end' => false,
             'stripe_invoice_paid' => false,

app/Services/DigitalOceanService.php

@@ -28,7 +28,7 @@ class DigitalOceanService
                 }
 
                 return $attempt * 100;
-            })
+            }, throw: false)
             ->{$method}($this->baseUrl.$endpoint, $data);
 
         if (! $response->successful()) {

app/Services/VultrService.php

@@ -17,7 +17,7 @@ class VultrService
             'Authorization' => 'Bearer '.$this->token,
         ])
             ->timeout(30)
-            ->retry(3, fn (int $attempt) => $attempt * 100)
+            ->retry(3, fn (int $attempt) => $attempt * 100, throw: false)
             ->{$method}($this->baseUrl.$endpoint, $data);
 
         if (! $response->successful()) {

app/Support/ClickhouseBackupCommand.php

@@ -0,0 +1,37 @@
+<?php
+
+namespace App\Support;
+
+final class ClickhouseBackupCommand
+{
+    /** @return array<int, string> */
+    public static function make(
+        string $containerName,
+        string $database,
+        string $archiveName,
+        string $backupDirectory,
+    ): array {
+        validateShellSafePath($database, 'database name');
+        validateFilenameSafe($archiveName, 'ClickHouse backup archive');
+
+        $backupDirectory = rtrim($backupDirectory, '/');
+        $containerBackupPath = '/var/lib/clickhouse/backups/'.$archiveName;
+        $backupLocation = $backupDirectory.'/'.$archiveName;
+        $query = "BACKUP DATABASE `{$database}` TO File('{$archiveName}')";
+
+        return [
+            'mkdir -p '.escapeshellarg($backupDirectory),
+            'docker exec '.escapeshellarg($containerName).' clickhouse-client --query '.escapeshellarg($query),
+            'docker cp '.escapeshellarg($containerName.':'.$containerBackupPath).' '.escapeshellarg($backupLocation),
+        ];
+    }
+
+    public static function cleanup(string $containerName, string $archiveName): string
+    {
+        validateFilenameSafe($archiveName, 'ClickHouse backup archive');
+
+        $containerBackupPath = '/var/lib/clickhouse/backups/'.$archiveName;
+
+        return 'docker exec '.escapeshellarg($containerName).' rm -f '.escapeshellarg($containerBackupPath);
+    }
+}