Merge remote-tracking branch 'origin/next' into feat/database-service-logs-endpoint

app/Actions/Application/CleanupPreviewDeployment.php

@@ -0,0 +1,176 @@
+<?php
+
+namespace App\Actions\Application;
+
+use App\Enums\ApplicationDeploymentStatus;
+use App\Jobs\DeleteResourceJob;
+use App\Models\Application;
+use App\Models\ApplicationDeploymentQueue;
+use App\Models\ApplicationPreview;
+use Lorisleiva\Actions\Concerns\AsAction;
+
+class CleanupPreviewDeployment
+{
+    use AsAction;
+
+    public string $jobQueue = 'high';
+
+    /**
+     * Clean up a PR preview deployment completely.
+     *
+     * This handles:
+     * 1. Cancelling active deployments for the PR (QUEUED/IN_PROGRESS → CANCELLED_BY_USER)
+     * 2. Killing helper containers by deployment_uuid
+     * 3. Stopping/removing all running PR containers
+     * 4. Dispatching DeleteResourceJob for thorough cleanup (volumes, networks, database records)
+     *
+     * This unifies the cleanup logic from GitHub webhook handler to be used across all providers.
+     */
+    public function handle(
+        Application $application,
+        int $pull_request_id,
+        ?ApplicationPreview $preview = null
+    ): array {
+        $result = [
+            'cancelled_deployments' => 0,
+            'killed_containers' => 0,
+            'status' => 'success',
+        ];
+
+        $server = $application->destination->server;
+
+        if (! $server->isFunctional()) {
+            return [
+                ...$result,
+                'status' => 'failed',
+                'message' => 'Server is not functional',
+            ];
+        }
+
+        // Step 1: Cancel all active deployments for this PR and kill helper containers
+        $result['cancelled_deployments'] = $this->cancelActiveDeployments(
+            $application,
+            $pull_request_id,
+            $server
+        );
+
+        // Step 2: Stop and remove all running PR containers
+        $result['killed_containers'] = $this->stopRunningContainers(
+            $application,
+            $pull_request_id,
+            $server
+        );
+
+        // Step 3: Find or use provided preview, then dispatch cleanup job for thorough cleanup
+        if (! $preview) {
+            $preview = ApplicationPreview::where('application_id', $application->id)
+                ->where('pull_request_id', $pull_request_id)
+                ->first();
+        }
+
+        if ($preview) {
+            DeleteResourceJob::dispatch($preview);
+        }
+
+        return $result;
+    }
+
+    /**
+     * Cancel all active (QUEUED/IN_PROGRESS) deployments for this PR.
+     */
+    private function cancelActiveDeployments(
+        Application $application,
+        int $pull_request_id,
+        $server
+    ): int {
+        $activeDeployments = ApplicationDeploymentQueue::where('application_id', $application->id)
+            ->where('pull_request_id', $pull_request_id)
+            ->whereIn('status', [
+                ApplicationDeploymentStatus::QUEUED->value,
+                ApplicationDeploymentStatus::IN_PROGRESS->value,
+            ])
+            ->get();
+
+        $cancelled = 0;
+        foreach ($activeDeployments as $deployment) {
+            try {
+                // Mark deployment as cancelled
+                $deployment->update([
+                    'status' => ApplicationDeploymentStatus::CANCELLED_BY_USER->value,
+                ]);
+
+                // Add cancellation log entry
+                $deployment->addLogEntry('Deployment cancelled: Pull request closed.', 'stderr');
+
+                // Try to kill helper container if it exists
+                $this->killHelperContainer($deployment->deployment_uuid, $server);
+                $cancelled++;
+            } catch (\Throwable $e) {
+                \Log::warning("Failed to cancel deployment {$deployment->id}: {$e->getMessage()}");
+            }
+        }
+
+        return $cancelled;
+    }
+
+    /**
+     * Kill the helper container used during deployment.
+     */
+    private function killHelperContainer(string $deployment_uuid, $server): void
+    {
+        try {
+            $escapedUuid = escapeshellarg($deployment_uuid);
+            $checkCommand = "docker ps -a --filter name={$escapedUuid} --format '{{.Names}}'";
+            $containerExists = instant_remote_process([$checkCommand], $server);
+
+            if ($containerExists && str($containerExists)->trim()->isNotEmpty()) {
+                instant_remote_process(["docker rm -f {$escapedUuid}"], $server);
+            }
+        } catch (\Throwable $e) {
+            // Silently handle - container may already be gone
+        }
+    }
+
+    /**
+     * Stop and remove all running containers for this PR.
+     */
+    private function stopRunningContainers(
+        Application $application,
+        int $pull_request_id,
+        $server
+    ): int {
+        $killed = 0;
+
+        try {
+            if ($server->isSwarm()) {
+                $escapedStackName = escapeshellarg("{$application->uuid}-{$pull_request_id}");
+                instant_remote_process(["docker stack rm {$escapedStackName}"], $server);
+                $killed++;
+            } else {
+                $containers = getCurrentApplicationContainerStatus(
+                    $server,
+                    $application->id,
+                    $pull_request_id
+                );
+
+                if ($containers->isNotEmpty()) {
+                    foreach ($containers as $container) {
+                        $containerName = data_get($container, 'Names');
+                        if ($containerName) {
+                            $escapedContainerName = escapeshellarg($containerName);
+                            instant_remote_process(
+                                ["docker rm -f {$escapedContainerName}"],
+                                $server
+                            );
+                            $killed++;
+                        }
+                    }
+                }
+            }
+        } catch (\Throwable $e) {
+            \Log::warning("Error stopping containers for PR #{$pull_request_id}: {$e->getMessage()}");
+        }
+
+        return $killed;
+    }
+}

app/Actions/Application/StopApplication.php

@@ -13,7 +13,7 @@ class StopApplication
 
     public string $jobQueue = 'high';
 
-    public function handle(Application $application, bool $previewDeployments = false, bool $dockerCleanup = true)
+    public function handle(Application $application, bool $previewDeployments = false, bool $dockerCleanup = true, bool $resetRestartCount = true)
     {
         $servers = collect([$application->destination->server]);
         if ($application?->additional_servers?->count() > 0) {
@@ -36,10 +36,11 @@ class StopApplication
                     : getCurrentApplicationContainerStatus($server, $application->id, 0);
 
                 $containersToStop = $containers->pluck('Names')->toArray();
+                $timeout = $application->settings->stopGracePeriodSeconds();
 
                 foreach ($containersToStop as $containerName) {
                     instant_remote_process(command: [
-                        "docker stop --time=30 $containerName",
+                        "docker stop --time=$timeout $containerName",
                         "docker rm -f $containerName",
                     ], server: $server, throwError: false);
                 }
@@ -55,6 +56,19 @@ class StopApplication
                 return $e->getMessage();
             }
         }
+
+        if ($resetRestartCount) {
+            $application->update([
+                'restart_count' => 0,
+                'last_restart_at' => null,
+                'last_restart_type' => null,
+            ]);
+        } else {
+            $application->update([
+                'status' => 'exited',
+            ]);
+        }
+
         ServiceStatusChanged::dispatch($application->environment->project->team->id);
     }
 }

app/Actions/Application/StopApplicationOneServer.php

@@ -20,13 +20,15 @@ class StopApplicationOneServer
         }
         try {
             $containers = getCurrentApplicationContainerStatus($server, $application->id, 0);
+            $timeout = $application->settings->stopGracePeriodSeconds();
+
             if ($containers->count() > 0) {
                 foreach ($containers as $container) {
                     $containerName = data_get($container, 'Names');
                     if ($containerName) {
                         instant_remote_process(
                             [
-                                "docker stop --time=30 $containerName",
+                                "docker stop --time=$timeout $containerName",
                                 "docker rm -f $containerName",
                             ],
                             $server

app/Actions/CoolifyTask/PrepareCoolifyTask.php

@@ -1,54 +0,0 @@
-<?php
-
-namespace App\Actions\CoolifyTask;
-
-use App\Data\CoolifyTaskArgs;
-use App\Jobs\CoolifyTask;
-use Spatie\Activitylog\Models\Activity;
-
-/**
- * The initial step to run a `CoolifyTask`: a remote SSH process
- * with monitoring/tracking/trace feature. Such thing is made
- * possible using an Activity model and some attributes.
- */
-class PrepareCoolifyTask
-{
-    protected Activity $activity;
-
-    protected CoolifyTaskArgs $remoteProcessArgs;
-
-    public function __construct(CoolifyTaskArgs $remoteProcessArgs)
-    {
-        $this->remoteProcessArgs = $remoteProcessArgs;
-
-        if ($remoteProcessArgs->model) {
-            $properties = $remoteProcessArgs->toArray();
-            unset($properties['model']);
-
-            $this->activity = activity()
-                ->withProperties($properties)
-                ->performedOn($remoteProcessArgs->model)
-                ->event($remoteProcessArgs->type)
-                ->log('[]');
-        } else {
-            $this->activity = activity()
-                ->withProperties($remoteProcessArgs->toArray())
-                ->event($remoteProcessArgs->type)
-                ->log('[]');
-        }
-    }
-
-    public function __invoke(): Activity
-    {
-        $job = new CoolifyTask(
-            activity: $this->activity,
-            ignore_errors: $this->remoteProcessArgs->ignore_errors,
-            call_event_on_finish: $this->remoteProcessArgs->call_event_on_finish,
-            call_event_data: $this->remoteProcessArgs->call_event_data,
-        );
-        dispatch($job);
-        $this->activity->refresh();
-
-        return $this->activity;
-    }
-}

app/Actions/Database/RestartDatabase.php

@@ -22,7 +22,7 @@ class RestartDatabase
         if (! $server->isFunctional()) {
             return 'Server is not functional';
         }
-        StopDatabase::run($database);
+        StopDatabase::run($database, dockerCleanup: false);
 
         return StartDatabase::run($database);
     }

app/Actions/Database/StartClickhouse.php

@@ -50,13 +50,9 @@ class StartClickhouse
                         ],
                     ],
                     'labels' => defaultDatabaseLabels($this->database)->toArray(),
-                    'healthcheck' => [
-                        'test' => "clickhouse-client --password {$this->database->clickhouse_admin_password} --query 'SELECT 1'",
-                        'interval' => '5s',
-                        'timeout' => '5s',
-                        'retries' => 10,
-                        'start_period' => '5s',
-                    ],
+                    'healthcheck' => $this->database->healthCheckConfiguration([
+                        'CMD', 'clickhouse-client', '--user', (string) $this->database->clickhouse_admin_user, '--password', (string) $this->database->clickhouse_admin_password, '--query', 'SELECT 1',
+                    ]),
                     'mem_limit' => $this->database->limits_memory,
                     'memswap_limit' => $this->database->limits_memory_swap,
                     'mem_swappiness' => $this->database->limits_memory_swappiness,
@@ -98,6 +94,9 @@ class StartClickhouse
         $docker_run_options = convertDockerRunToCompose($this->database->custom_docker_run_options);
         $docker_compose = generateCustomDockerRunOptionsForDatabases($docker_run_options, $docker_compose, $container_name, $this->database->destination->network);
 
+        if (! $this->database->isHealthcheckEnabled()) {
+            unset($docker_compose['services'][$container_name]['healthcheck']);
+        }
         $docker_compose = Yaml::dump($docker_compose, 10);
         $docker_compose_base64 = base64_encode($docker_compose);
         $this->commands[] = "echo '{$docker_compose_base64}' | base64 -d | tee $this->configuration_dir/docker-compose.yml > /dev/null";
@@ -105,6 +104,8 @@ class StartClickhouse
         $this->commands[] = "echo '{$readme}' > $this->configuration_dir/README.md";
         $this->commands[] = "echo 'Pulling {$database->image} image.'";
         $this->commands[] = "docker compose -f $this->configuration_dir/docker-compose.yml pull";
+        $this->commands[] = "docker stop -t 10 $container_name 2>/dev/null || true";
+        $this->commands[] = "docker rm -f $container_name 2>/dev/null || true";
         $this->commands[] = "docker compose -f $this->configuration_dir/docker-compose.yml up -d";
         $this->commands[] = "echo 'Database started.'";
 
@@ -150,12 +151,16 @@ class StartClickhouse
             $environment_variables->push("$env->key=$env->real_value");
         }
 
-        if ($environment_variables->filter(fn ($env) => str($env)->contains('CLICKHOUSE_ADMIN_USER'))->isEmpty()) {
-            $environment_variables->push("CLICKHOUSE_ADMIN_USER={$this->database->clickhouse_admin_user}");
+        if ($environment_variables->filter(fn ($env) => str($env)->contains('CLICKHOUSE_USER'))->isEmpty()) {
+            $environment_variables->push("CLICKHOUSE_USER={$this->database->clickhouse_admin_user}");
         }
 
-        if ($environment_variables->filter(fn ($env) => str($env)->contains('CLICKHOUSE_ADMIN_PASSWORD'))->isEmpty()) {
-            $environment_variables->push("CLICKHOUSE_ADMIN_PASSWORD={$this->database->clickhouse_admin_password}");
+        if ($environment_variables->filter(fn ($env) => str($env)->contains('CLICKHOUSE_PASSWORD'))->isEmpty()) {
+            $environment_variables->push("CLICKHOUSE_PASSWORD={$this->database->clickhouse_admin_password}");
+        }
+
+        if ($environment_variables->filter(fn ($env) => str($env)->contains('CLICKHOUSE_DB'))->isEmpty()) {
+            $environment_variables->push("CLICKHOUSE_DB={$this->database->clickhouse_db}");
         }
 
         add_coolify_default_environment_variables($this->database, $environment_variables, $environment_variables);

app/Actions/Database/StartDatabase.php

@@ -11,12 +11,16 @@ use App\Models\StandaloneMysql;
 use App\Models\StandalonePostgresql;
 use App\Models\StandaloneRedis;
 use Lorisleiva\Actions\Concerns\AsAction;
+use Lorisleiva\Actions\Decorators\JobDecorator;
 
 class StartDatabase
 {
     use AsAction;
 
-    public string $jobQueue = 'high';
+    public function configureJob(JobDecorator $job): void
+    {
+        $job->onQueue(deployment_queue());
+    }
 
     public function handle(StandaloneRedis|StandalonePostgresql|StandaloneMongodb|StandaloneMysql|StandaloneMariadb|StandaloneKeydb|StandaloneDragonfly|StandaloneClickhouse $database)
     {
@@ -25,28 +29,28 @@ class StartDatabase
             return 'Server is not functional';
         }
         switch ($database->getMorphClass()) {
-            case \App\Models\StandalonePostgresql::class:
+            case StandalonePostgresql::class:
                 $activity = StartPostgresql::run($database);
                 break;
-            case \App\Models\StandaloneRedis::class:
+            case StandaloneRedis::class:
                 $activity = StartRedis::run($database);
                 break;
-            case \App\Models\StandaloneMongodb::class:
+            case StandaloneMongodb::class:
                 $activity = StartMongodb::run($database);
                 break;
-            case \App\Models\StandaloneMysql::class:
+            case StandaloneMysql::class:
                 $activity = StartMysql::run($database);
                 break;
-            case \App\Models\StandaloneMariadb::class:
+            case StandaloneMariadb::class:
                 $activity = StartMariadb::run($database);
                 break;
-            case \App\Models\StandaloneKeydb::class:
+            case StandaloneKeydb::class:
                 $activity = StartKeydb::run($database);
                 break;
-            case \App\Models\StandaloneDragonfly::class:
+            case StandaloneDragonfly::class:
                 $activity = StartDragonfly::run($database);
                 break;
-            case \App\Models\StandaloneClickhouse::class:
+            case StandaloneClickhouse::class:
                 $activity = StartClickhouse::run($database);
                 break;
         }

app/Actions/Database/StartDatabaseProxy.php

@@ -11,14 +11,19 @@ use App\Models\StandaloneMongodb;
 use App\Models\StandaloneMysql;
 use App\Models\StandalonePostgresql;
 use App\Models\StandaloneRedis;
+use App\Notifications\Container\ContainerRestarted;
 use Lorisleiva\Actions\Concerns\AsAction;
+use Lorisleiva\Actions\Decorators\JobDecorator;
 use Symfony\Component\Yaml\Yaml;
 
 class StartDatabaseProxy
 {
     use AsAction;
 
-    public string $jobQueue = 'high';
+    public function configureJob(JobDecorator $job): void
+    {
+        $job->onQueue(deployment_queue());
+    }
 
     public function handle(StandaloneRedis|StandalonePostgresql|StandaloneMongodb|StandaloneMysql|StandaloneMariadb|StandaloneKeydb|StandaloneDragonfly|StandaloneClickhouse|ServiceDatabase $database)
     {
@@ -29,11 +34,10 @@ class StartDatabaseProxy
         $proxyContainerName = "{$database->uuid}-proxy";
         $isSSLEnabled = $database->enable_ssl ?? false;
 
-        if ($database->getMorphClass() === \App\Models\ServiceDatabase::class) {
+        if ($database->getMorphClass() === ServiceDatabase::class) {
             $databaseType = $database->databaseType();
             $network = $database->service->uuid;
             $server = data_get($database, 'service.destination.server');
-            $proxyContainerName = "{$database->service->uuid}-proxy";
             $containerName = "{$database->name}-{$database->service->uuid}";
         }
         $internalPort = match ($databaseType) {
@@ -52,9 +56,11 @@ class StartDatabaseProxy
         }
 
         $configuration_dir = database_proxy_dir($database->uuid);
+        $host_configuration_dir = $configuration_dir;
         if (isDev()) {
-            $configuration_dir = '/var/lib/docker/volumes/coolify_dev_coolify_data/_data/databases/'.$database->uuid.'/proxy';
+            $host_configuration_dir = '/var/lib/docker/volumes/coolify_dev_coolify_data/_data/databases/'.$database->uuid.'/proxy';
         }
+        $timeoutConfig = $this->buildProxyTimeoutConfig($database->public_port_timeout);
         $nginxconf = <<<EOF
     user  nginx;
     worker_processes  auto;
@@ -68,6 +74,7 @@ class StartDatabaseProxy
        server {
             listen $database->public_port;
             proxy_pass $containerName:$internalPort;
+            $timeoutConfig
        }
     }
     EOF;
@@ -86,7 +93,7 @@ class StartDatabaseProxy
                     'volumes' => [
                         [
                             'type' => 'bind',
-                            'source' => "$configuration_dir/nginx.conf",
+                            'source' => "$host_configuration_dir/nginx.conf",
                             'target' => '/etc/nginx/nginx.conf',
                         ],
                     ],
@@ -113,12 +120,59 @@ class StartDatabaseProxy
         $dockercompose_base64 = base64_encode(Yaml::dump($docker_compose, 4, 2));
         $nginxconf_base64 = base64_encode($nginxconf);
         instant_remote_process(["docker rm -f $proxyContainerName"], $server, false);
-        instant_remote_process([
-            "mkdir -p $configuration_dir",
-            "echo '{$nginxconf_base64}' | base64 -d | tee $configuration_dir/nginx.conf > /dev/null",
-            "echo '{$dockercompose_base64}' | base64 -d | tee $configuration_dir/docker-compose.yaml > /dev/null",
-            "docker compose --project-directory {$configuration_dir} pull",
-            "docker compose --project-directory {$configuration_dir} up -d",
-        ], $server);
+
+        try {
+            instant_remote_process([
+                "mkdir -p $configuration_dir",
+                "echo '{$nginxconf_base64}' | base64 -d | tee $configuration_dir/nginx.conf > /dev/null",
+                "echo '{$dockercompose_base64}' | base64 -d | tee $configuration_dir/docker-compose.yaml > /dev/null",
+                "docker compose --project-directory {$configuration_dir} pull",
+                "docker compose --project-directory {$configuration_dir} up -d",
+            ], $server);
+        } catch (\RuntimeException $e) {
+            if ($this->isNonTransientError($e->getMessage())) {
+                $database->update(['is_public' => false]);
+
+                $team = data_get($database, 'environment.project.team')
+                    ?? data_get($database, 'service.environment.project.team');
+
+                $team?->notify(
+                    new ContainerRestarted(
+                        "TCP Proxy for {$database->name} database has been disabled due to error: {$e->getMessage()}",
+                        $server,
+                    )
+                );
+
+                return;
+            }
+
+            throw $e;
+        }
+    }
+
+    private function isNonTransientError(string $message): bool
+    {
+        $nonTransientPatterns = [
+            'port is already allocated',
+            'address already in use',
+            'Bind for',
+        ];
+
+        foreach ($nonTransientPatterns as $pattern) {
+            if (str_contains($message, $pattern)) {
+                return true;
+            }
+        }
+
+        return false;
+    }
+
+    private function buildProxyTimeoutConfig(?int $timeout): string
+    {
+        if ($timeout === null || $timeout < 1) {
+            $timeout = 3600;
+        }
+
+        return "proxy_timeout {$timeout}s;";
     }
 }

app/Actions/Database/StartDragonfly.php

@@ -55,11 +55,11 @@ class StartDragonfly
             $this->commands[] = "mkdir -p $this->configuration_dir/ssl";
 
             $server = $this->database->destination->server;
-            $caCert = SslCertificate::where('server_id', $server->id)->where('is_ca_certificate', true)->first();
+            $caCert = $server->sslCertificates()->where('is_ca_certificate', true)->first();
 
             if (! $caCert) {
                 $server->generateCaCertificate();
-                $caCert = SslCertificate::where('server_id', $server->id)->where('is_ca_certificate', true)->first();
+                $caCert = $server->sslCertificates()->where('is_ca_certificate', true)->first();
             }
 
             if (! $caCert) {
@@ -106,13 +106,9 @@ class StartDragonfly
                         $this->database->destination->network,
                     ],
                     'labels' => defaultDatabaseLabels($this->database)->toArray(),
-                    'healthcheck' => [
-                        'test' => "redis-cli -a {$this->database->dragonfly_password} ping",
-                        'interval' => '5s',
-                        'timeout' => '5s',
-                        'retries' => 10,
-                        'start_period' => '5s',
-                    ],
+                    'healthcheck' => $this->database->healthCheckConfiguration([
+                        'CMD', 'redis-cli', '-a', (string) $this->database->dragonfly_password, 'ping',
+                    ]),
                     'mem_limit' => $this->database->limits_memory,
                     'memswap_limit' => $this->database->limits_memory_swap,
                     'mem_swappiness' => $this->database->limits_memory_swappiness,
@@ -182,6 +178,9 @@ class StartDragonfly
         $docker_run_options = convertDockerRunToCompose($this->database->custom_docker_run_options);
         $docker_compose = generateCustomDockerRunOptionsForDatabases($docker_run_options, $docker_compose, $container_name, $this->database->destination->network);
 
+        if (! $this->database->isHealthcheckEnabled()) {
+            unset($docker_compose['services'][$container_name]['healthcheck']);
+        }
         $docker_compose = Yaml::dump($docker_compose, 10);
         $docker_compose_base64 = base64_encode($docker_compose);
         $this->commands[] = "echo '{$docker_compose_base64}' | base64 -d | tee $this->configuration_dir/docker-compose.yml > /dev/null";
@@ -192,6 +191,8 @@ class StartDragonfly
         if ($this->database->enable_ssl) {
             $this->commands[] = "chown -R 999:999 $this->configuration_dir/ssl/server.key $this->configuration_dir/ssl/server.crt";
         }
+        $this->commands[] = "docker stop -t 10 $container_name 2>/dev/null || true";
+        $this->commands[] = "docker rm -f $container_name 2>/dev/null || true";
         $this->commands[] = "docker compose -f $this->configuration_dir/docker-compose.yml up -d";
         $this->commands[] = "echo 'Database started.'";
 

app/Actions/Database/StartKeydb.php

@@ -5,7 +5,6 @@ namespace App\Actions\Database;
 use App\Helpers\SslHelper;
 use App\Models\SslCertificate;
 use App\Models\StandaloneKeydb;
-use Illuminate\Support\Facades\Storage;
 use Lorisleiva\Actions\Concerns\AsAction;
 use Symfony\Component\Yaml\Yaml;
 
@@ -56,11 +55,11 @@ class StartKeydb
             $this->commands[] = "mkdir -p $this->configuration_dir/ssl";
 
             $server = $this->database->destination->server;
-            $caCert = SslCertificate::where('server_id', $server->id)->where('is_ca_certificate', true)->first();
+            $caCert = $server->sslCertificates()->where('is_ca_certificate', true)->first();
 
             if (! $caCert) {
                 $server->generateCaCertificate();
-                $caCert = SslCertificate::where('server_id', $server->id)->where('is_ca_certificate', true)->first();
+                $caCert = $server->sslCertificates()->where('is_ca_certificate', true)->first();
             }
 
             if (! $caCert) {
@@ -109,13 +108,9 @@ class StartKeydb
                         $this->database->destination->network,
                     ],
                     'labels' => defaultDatabaseLabels($this->database)->toArray(),
-                    'healthcheck' => [
-                        'test' => "keydb-cli --pass {$this->database->keydb_password} ping",
-                        'interval' => '5s',
-                        'timeout' => '5s',
-                        'retries' => 10,
-                        'start_period' => '5s',
-                    ],
+                    'healthcheck' => $this->database->healthCheckConfiguration([
+                        'CMD', 'keydb-cli', '--pass', (string) $this->database->keydb_password, 'ping',
+                    ]),
                     'mem_limit' => $this->database->limits_memory,
                     'memswap_limit' => $this->database->limits_memory_swap,
                     'mem_swappiness' => $this->database->limits_memory_swappiness,
@@ -167,7 +162,7 @@ class StartKeydb
             $docker_compose['volumes'] = $volume_names;
         }
 
-        if (! is_null($this->database->keydb_conf) || ! empty($this->database->keydb_conf)) {
+        if (! is_null($this->database->keydb_conf) && ! empty($this->database->keydb_conf)) {
             $docker_compose['services'][$container_name]['volumes'] = array_merge(
                 $docker_compose['services'][$container_name]['volumes'] ?? [],
                 [
@@ -198,6 +193,9 @@ class StartKeydb
         // Add custom docker run options
         $docker_run_options = convertDockerRunToCompose($this->database->custom_docker_run_options);
         $docker_compose = generateCustomDockerRunOptionsForDatabases($docker_run_options, $docker_compose, $container_name, $this->database->destination->network);
+        if (! $this->database->isHealthcheckEnabled()) {
+            unset($docker_compose['services'][$container_name]['healthcheck']);
+        }
         $docker_compose = Yaml::dump($docker_compose, 10);
         $docker_compose_base64 = base64_encode($docker_compose);
         $this->commands[] = "echo '{$docker_compose_base64}' | base64 -d | tee $this->configuration_dir/docker-compose.yml > /dev/null";
@@ -208,6 +206,11 @@ class StartKeydb
         if ($this->database->enable_ssl) {
             $this->commands[] = "chown -R 999:999 $this->configuration_dir/ssl/server.key $this->configuration_dir/ssl/server.crt";
         }
+        if (! is_null($this->database->keydb_conf) && ! empty($this->database->keydb_conf)) {
+            $this->commands[] = "chown 999:999 $this->configuration_dir/keydb.conf";
+        }
+        $this->commands[] = "docker stop -t 10 $container_name 2>/dev/null || true";
+        $this->commands[] = "docker rm -f $container_name 2>/dev/null || true";
         $this->commands[] = "docker compose -f $this->configuration_dir/docker-compose.yml up -d";
         $this->commands[] = "echo 'Database started.'";
 
@@ -268,10 +271,9 @@ class StartKeydb
             return;
         }
         $filename = 'keydb.conf';
-        Storage::disk('local')->put("tmp/keydb.conf_{$this->database->uuid}", $this->database->keydb_conf);
-        $path = Storage::path("tmp/keydb.conf_{$this->database->uuid}");
-        instant_scp($path, "{$this->configuration_dir}/{$filename}", $this->database->destination->server);
-        Storage::disk('local')->delete("tmp/keydb.conf_{$this->database->uuid}");
+        $content = $this->database->keydb_conf;
+        $content_base64 = base64_encode($content);
+        $this->commands[] = "echo '{$content_base64}' | base64 -d | tee $this->configuration_dir/{$filename} > /dev/null";
     }
 
     private function buildStartCommand(): string

app/Actions/Database/StartMariadb.php

@@ -57,11 +57,11 @@ class StartMariadb
             $this->commands[] = "mkdir -p $this->configuration_dir/ssl";
 
             $server = $this->database->destination->server;
-            $caCert = SslCertificate::where('server_id', $server->id)->where('is_ca_certificate', true)->first();
+            $caCert = $server->sslCertificates()->where('is_ca_certificate', true)->first();
 
             if (! $caCert) {
                 $server->generateCaCertificate();
-                $caCert = SslCertificate::where('server_id', $server->id)->where('is_ca_certificate', true)->first();
+                $caCert = $server->sslCertificates()->where('is_ca_certificate', true)->first();
             }
 
             if (! $caCert) {
@@ -103,13 +103,9 @@ class StartMariadb
                         $this->database->destination->network,
                     ],
                     'labels' => defaultDatabaseLabels($this->database)->toArray(),
-                    'healthcheck' => [
-                        'test' => ['CMD', 'healthcheck.sh', '--connect', '--innodb_initialized'],
-                        'interval' => '5s',
-                        'timeout' => '5s',
-                        'retries' => 10,
-                        'start_period' => '5s',
-                    ],
+                    'healthcheck' => $this->database->healthCheckConfiguration([
+                        'CMD', 'healthcheck.sh', '--connect', '--innodb_initialized',
+                    ]),
                     'mem_limit' => $this->database->limits_memory,
                     'memswap_limit' => $this->database->limits_memory_swap,
                     'mem_swappiness' => $this->database->limits_memory_swappiness,
@@ -175,7 +171,7 @@ class StartMariadb
             );
         }
 
-        if (! is_null($this->database->mariadb_conf) || ! empty($this->database->mariadb_conf)) {
+        if (! is_null($this->database->mariadb_conf) && ! empty($this->database->mariadb_conf)) {
             $docker_compose['services'][$container_name]['volumes'] = array_merge(
                 $docker_compose['services'][$container_name]['volumes'],
                 [
@@ -202,6 +198,9 @@ class StartMariadb
             ];
         }
 
+        if (! $this->database->isHealthcheckEnabled()) {
+            unset($docker_compose['services'][$container_name]['healthcheck']);
+        }
         $docker_compose = Yaml::dump($docker_compose, 10);
         $docker_compose_base64 = base64_encode($docker_compose);
         $this->commands[] = "echo '{$docker_compose_base64}' | base64 -d | tee $this->configuration_dir/docker-compose.yml > /dev/null";
@@ -209,6 +208,8 @@ class StartMariadb
         $this->commands[] = "echo '{$readme}' > $this->configuration_dir/README.md";
         $this->commands[] = "echo 'Pulling {$database->image} image.'";
         $this->commands[] = "docker compose -f $this->configuration_dir/docker-compose.yml pull";
+        $this->commands[] = "docker stop -t 10 $container_name 2>/dev/null || true";
+        $this->commands[] = "docker rm -f $container_name 2>/dev/null || true";
         $this->commands[] = "docker compose -f $this->configuration_dir/docker-compose.yml up -d";
         $this->commands[] = "echo 'Database started.'";
         if ($this->database->enable_ssl) {

app/Actions/Database/StartMongodb.php

@@ -61,11 +61,11 @@ class StartMongodb
             $this->commands[] = "mkdir -p $this->configuration_dir/ssl";
 
             $server = $this->database->destination->server;
-            $caCert = SslCertificate::where('server_id', $server->id)->where('is_ca_certificate', true)->first();
+            $caCert = $server->sslCertificates()->where('is_ca_certificate', true)->first();
 
             if (! $caCert) {
                 $server->generateCaCertificate();
-                $caCert = SslCertificate::where('server_id', $server->id)->where('is_ca_certificate', true)->first();
+                $caCert = $server->sslCertificates()->where('is_ca_certificate', true)->first();
             }
 
             if (! $caCert) {
@@ -109,17 +109,11 @@ class StartMongodb
                         $this->database->destination->network,
                     ],
                     'labels' => defaultDatabaseLabels($this->database)->toArray(),
-                    'healthcheck' => [
-                        'test' => [
-                            'CMD',
-                            'echo',
-                            'ok',
-                        ],
-                        'interval' => '5s',
-                        'timeout' => '5s',
-                        'retries' => 10,
-                        'start_period' => '5s',
-                    ],
+                    'healthcheck' => $this->database->healthCheckConfiguration([
+                        'CMD',
+                        'echo',
+                        'ok',
+                    ]),
                     'mem_limit' => $this->database->limits_memory,
                     'memswap_limit' => $this->database->limits_memory_swap,
                     'mem_swappiness' => $this->database->limits_memory_swappiness,
@@ -253,6 +247,9 @@ class StartMongodb
             $docker_compose['services'][$container_name]['command'] = $commandParts;
         }
 
+        if (! $this->database->isHealthcheckEnabled()) {
+            unset($docker_compose['services'][$container_name]['healthcheck']);
+        }
         $docker_compose = Yaml::dump($docker_compose, 10);
         $docker_compose_base64 = base64_encode($docker_compose);
         $this->commands[] = "echo '{$docker_compose_base64}' | base64 -d | tee $this->configuration_dir/docker-compose.yml > /dev/null";
@@ -260,6 +257,8 @@ class StartMongodb
         $this->commands[] = "echo '{$readme}' > $this->configuration_dir/README.md";
         $this->commands[] = "echo 'Pulling {$database->image} image.'";
         $this->commands[] = "docker compose -f $this->configuration_dir/docker-compose.yml pull";
+        $this->commands[] = "docker stop -t 10 $container_name 2>/dev/null || true";
+        $this->commands[] = "docker rm -f $container_name 2>/dev/null || true";
         $this->commands[] = "docker compose -f $this->configuration_dir/docker-compose.yml up -d";
         if ($this->database->enable_ssl) {
             $this->commands[] = executeInDocker($this->database->uuid, 'chown mongodb:mongodb /etc/mongo/certs/server.pem');
@@ -338,7 +337,10 @@ class StartMongodb
 
     private function add_default_database()
     {
-        $content = "db = db.getSiblingDB(\"{$this->database->mongo_initdb_database}\");db.createCollection('init_collection');db.createUser({user: \"{$this->database->mongo_initdb_root_username}\", pwd: \"{$this->database->mongo_initdb_root_password}\",roles: [{role:\"readWrite\",db:\"{$this->database->mongo_initdb_database}\"}]});";
+        $dbJson = json_encode($this->database->mongo_initdb_database, JSON_UNESCAPED_SLASHES);
+        $userJson = json_encode($this->database->mongo_initdb_root_username, JSON_UNESCAPED_SLASHES);
+        $pwdJson = json_encode($this->database->mongo_initdb_root_password, JSON_UNESCAPED_SLASHES);
+        $content = "db = db.getSiblingDB({$dbJson});db.createCollection('init_collection');db.createUser({user: {$userJson}, pwd: {$pwdJson}, roles: [{role:\"readWrite\",db:{$dbJson}}]});";
         $content_base64 = base64_encode($content);
         $this->commands[] = "mkdir -p $this->configuration_dir/docker-entrypoint-initdb.d";
         $this->commands[] = "echo '{$content_base64}' | base64 -d | tee $this->configuration_dir/docker-entrypoint-initdb.d/01-default-database.js > /dev/null";

app/Actions/Database/StartMysql.php

@@ -57,11 +57,11 @@ class StartMysql
             $this->commands[] = "mkdir -p $this->configuration_dir/ssl";
 
             $server = $this->database->destination->server;
-            $caCert = SslCertificate::where('server_id', $server->id)->where('is_ca_certificate', true)->first();
+            $caCert = $server->sslCertificates()->where('is_ca_certificate', true)->first();
 
             if (! $caCert) {
                 $server->generateCaCertificate();
-                $caCert = SslCertificate::where('server_id', $server->id)->where('is_ca_certificate', true)->first();
+                $caCert = $server->sslCertificates()->where('is_ca_certificate', true)->first();
             }
 
             if (! $caCert) {
@@ -103,13 +103,9 @@ class StartMysql
                         $this->database->destination->network,
                     ],
                     'labels' => defaultDatabaseLabels($this->database)->toArray(),
-                    'healthcheck' => [
-                        'test' => ['CMD', 'mysqladmin', 'ping', '-h', 'localhost', '-u', 'root', "-p{$this->database->mysql_root_password}"],
-                        'interval' => '5s',
-                        'timeout' => '5s',
-                        'retries' => 10,
-                        'start_period' => '5s',
-                    ],
+                    'healthcheck' => $this->database->healthCheckConfiguration([
+                        'CMD', 'mysqladmin', 'ping', '-h', 'localhost', '-u', 'root', "-p{$this->database->mysql_root_password}",
+                    ]),
                     'mem_limit' => $this->database->limits_memory,
                     'memswap_limit' => $this->database->limits_memory_swap,
                     'mem_swappiness' => $this->database->limits_memory_swappiness,
@@ -175,7 +171,7 @@ class StartMysql
             );
         }
 
-        if (! is_null($this->database->mysql_conf) || ! empty($this->database->mysql_conf)) {
+        if (! is_null($this->database->mysql_conf) && ! empty($this->database->mysql_conf)) {
             $docker_compose['services'][$container_name]['volumes'] = array_merge(
                 $docker_compose['services'][$container_name]['volumes'] ?? [],
                 [
@@ -203,6 +199,9 @@ class StartMysql
             ];
         }
 
+        if (! $this->database->isHealthcheckEnabled()) {
+            unset($docker_compose['services'][$container_name]['healthcheck']);
+        }
         $docker_compose = Yaml::dump($docker_compose, 10);
         $docker_compose_base64 = base64_encode($docker_compose);
         $this->commands[] = "echo '{$docker_compose_base64}' | base64 -d | tee $this->configuration_dir/docker-compose.yml > /dev/null";
@@ -210,10 +209,13 @@ class StartMysql
         $this->commands[] = "echo '{$readme}' > $this->configuration_dir/README.md";
         $this->commands[] = "echo 'Pulling {$database->image} image.'";
         $this->commands[] = "docker compose -f $this->configuration_dir/docker-compose.yml pull";
+        $this->commands[] = "docker stop -t 10 $container_name 2>/dev/null || true";
+        $this->commands[] = "docker rm -f $container_name 2>/dev/null || true";
         $this->commands[] = "docker compose -f $this->configuration_dir/docker-compose.yml up -d";
 
         if ($this->database->enable_ssl) {
-            $this->commands[] = executeInDocker($this->database->uuid, "chown {$this->database->mysql_user}:{$this->database->mysql_user} /etc/mysql/certs/server.crt /etc/mysql/certs/server.key");
+            $mysqlUser = escapeshellarg($this->database->mysql_user);
+            $this->commands[] = executeInDocker($this->database->uuid, "chown {$mysqlUser}:{$mysqlUser} /etc/mysql/certs/server.crt /etc/mysql/certs/server.key");
         }
 
         $this->commands[] = "echo 'Database started.'";

app/Actions/Database/StartPostgresql.php

@@ -62,11 +62,11 @@ class StartPostgresql
             $this->commands[] = "mkdir -p $this->configuration_dir/ssl";
 
             $server = $this->database->destination->server;
-            $caCert = SslCertificate::where('server_id', $server->id)->where('is_ca_certificate', true)->first();
+            $caCert = $server->sslCertificates()->where('is_ca_certificate', true)->first();
 
             if (! $caCert) {
                 $server->generateCaCertificate();
-                $caCert = SslCertificate::where('server_id', $server->id)->where('is_ca_certificate', true)->first();
+                $caCert = $server->sslCertificates()->where('is_ca_certificate', true)->first();
             }
 
             if (! $caCert) {
@@ -110,16 +110,9 @@ class StartPostgresql
                         $this->database->destination->network,
                     ],
                     'labels' => defaultDatabaseLabels($this->database)->toArray(),
-                    'healthcheck' => [
-                        'test' => [
-                            'CMD-SHELL',
-                            "psql -U {$this->database->postgres_user} -d {$this->database->postgres_db} -c 'SELECT 1' || exit 1",
-                        ],
-                        'interval' => '5s',
-                        'timeout' => '5s',
-                        'retries' => 10,
-                        'start_period' => '5s',
-                    ],
+                    'healthcheck' => $this->database->healthCheckConfiguration([
+                        'CMD', 'psql', '-U', (string) $this->database->postgres_user, '-d', (string) $this->database->postgres_db, '-c', 'SELECT 1',
+                    ]),
                     'mem_limit' => $this->database->limits_memory,
                     'memswap_limit' => $this->database->limits_memory_swap,
                     'mem_swappiness' => $this->database->limits_memory_swappiness,
@@ -185,6 +178,8 @@ class StartPostgresql
             }
         }
 
+        $command = ['postgres'];
+
         if (filled($this->database->postgres_conf)) {
             $docker_compose['services'][$container_name]['volumes'] = array_merge(
                 $docker_compose['services'][$container_name]['volumes'],
@@ -195,29 +190,28 @@ class StartPostgresql
                     'read_only' => true,
                 ]]
             );
-            $docker_compose['services'][$container_name]['command'] = [
-                'postgres',
-                '-c',
-                'config_file=/etc/postgresql/postgresql.conf',
-            ];
+            $command = array_merge($command, ['-c', 'config_file=/etc/postgresql/postgresql.conf']);
         }
 
         if ($this->database->enable_ssl) {
-            $docker_compose['services'][$container_name]['command'] = [
-                'postgres',
-                '-c',
-                'ssl=on',
-                '-c',
-                'ssl_cert_file=/var/lib/postgresql/certs/server.crt',
-                '-c',
-                'ssl_key_file=/var/lib/postgresql/certs/server.key',
-            ];
+            $command = array_merge($command, [
+                '-c', 'ssl=on',
+                '-c', 'ssl_cert_file=/var/lib/postgresql/certs/server.crt',
+                '-c', 'ssl_key_file=/var/lib/postgresql/certs/server.key',
+            ]);
         }
 
         // Add custom docker run options
         $docker_run_options = convertDockerRunToCompose($this->database->custom_docker_run_options);
         $docker_compose = generateCustomDockerRunOptionsForDatabases($docker_run_options, $docker_compose, $container_name, $this->database->destination->network);
 
+        if (count($command) > 1) {
+            $docker_compose['services'][$container_name]['command'] = $command;
+        }
+
+        if (! $this->database->isHealthcheckEnabled()) {
+            unset($docker_compose['services'][$container_name]['healthcheck']);
+        }
         $docker_compose = Yaml::dump($docker_compose, 10);
         $docker_compose_base64 = base64_encode($docker_compose);
         $this->commands[] = "echo '{$docker_compose_base64}' | base64 -d | tee $this->configuration_dir/docker-compose.yml > /dev/null";
@@ -225,9 +219,12 @@ class StartPostgresql
         $this->commands[] = "echo '{$readme}' > $this->configuration_dir/README.md";
         $this->commands[] = "echo 'Pulling {$database->image} image.'";
         $this->commands[] = "docker compose -f $this->configuration_dir/docker-compose.yml pull";
+        $this->commands[] = "docker stop -t 10 $container_name 2>/dev/null || true";
+        $this->commands[] = "docker rm -f $container_name 2>/dev/null || true";
         $this->commands[] = "docker compose -f $this->configuration_dir/docker-compose.yml up -d";
         if ($this->database->enable_ssl) {
-            $this->commands[] = executeInDocker($this->database->uuid, "chown {$this->database->postgres_user}:{$this->database->postgres_user} /var/lib/postgresql/certs/server.key /var/lib/postgresql/certs/server.crt");
+            $postgresUser = escapeshellarg($this->database->postgres_user);
+            $this->commands[] = executeInDocker($this->database->uuid, "chown {$postgresUser}:{$postgresUser} /var/lib/postgresql/certs/server.key /var/lib/postgresql/certs/server.crt");
         }
         $this->commands[] = "echo 'Database started.'";
 
@@ -304,9 +301,18 @@ class StartPostgresql
         foreach ($this->database->init_scripts as $init_script) {
             $filename = data_get($init_script, 'filename');
             $content = data_get($init_script, 'content');
+
+            // Normalise filename without rejecting legacy values so previously created
+            // init scripts keep deploying. basename() strips any directory components
+            // (path traversal) and escapeshellarg() contains every shell metacharacter
+            // in the tee target. Livewire / API validate new filenames up front.
+            $filename = basename((string) $filename);
+
+            $target_path = "$this->configuration_dir/docker-entrypoint-initdb.d/{$filename}";
+            $escaped_target = escapeshellarg($target_path);
             $content_base64 = base64_encode($content);
-            $this->commands[] = "echo '{$content_base64}' | base64 -d | tee $this->configuration_dir/docker-entrypoint-initdb.d/{$filename} > /dev/null";
-            $this->init_scripts[] = "$this->configuration_dir/docker-entrypoint-initdb.d/{$filename}";
+            $this->commands[] = "echo '{$content_base64}' | base64 -d | tee {$escaped_target} > /dev/null";
+            $this->init_scripts[] = $target_path;
         }
     }
 

app/Actions/Database/StartRedis.php

@@ -5,7 +5,6 @@ namespace App\Actions\Database;
 use App\Helpers\SslHelper;
 use App\Models\SslCertificate;
 use App\Models\StandaloneRedis;
-use Illuminate\Support\Facades\Storage;
 use Lorisleiva\Actions\Concerns\AsAction;
 use Symfony\Component\Yaml\Yaml;
 
@@ -56,11 +55,11 @@ class StartRedis
             $this->commands[] = "mkdir -p $this->configuration_dir/ssl";
 
             $server = $this->database->destination->server;
-            $caCert = SslCertificate::where('server_id', $server->id)->where('is_ca_certificate', true)->first();
+            $caCert = $server->sslCertificates()->where('is_ca_certificate', true)->first();
 
             if (! $caCert) {
                 $server->generateCaCertificate();
-                $caCert = SslCertificate::where('server_id', $server->id)->where('is_ca_certificate', true)->first();
+                $caCert = $server->sslCertificates()->where('is_ca_certificate', true)->first();
             }
 
             if (! $caCert) {
@@ -106,17 +105,11 @@ class StartRedis
                         $this->database->destination->network,
                     ],
                     'labels' => defaultDatabaseLabels($this->database)->toArray(),
-                    'healthcheck' => [
-                        'test' => [
-                            'CMD-SHELL',
-                            'redis-cli',
-                            'ping',
-                        ],
-                        'interval' => '5s',
-                        'timeout' => '5s',
-                        'retries' => 10,
-                        'start_period' => '5s',
-                    ],
+                    'healthcheck' => $this->database->healthCheckConfiguration([
+                        'CMD-SHELL',
+                        'redis-cli',
+                        'ping',
+                    ]),
                     'mem_limit' => $this->database->limits_memory,
                     'memswap_limit' => $this->database->limits_memory_swap,
                     'mem_swappiness' => $this->database->limits_memory_swappiness,
@@ -182,7 +175,7 @@ class StartRedis
             );
         }
 
-        if (! is_null($this->database->redis_conf) || ! empty($this->database->redis_conf)) {
+        if (! is_null($this->database->redis_conf) && ! empty($this->database->redis_conf)) {
             $docker_compose['services'][$container_name]['volumes'][] = [
                 'type' => 'bind',
                 'source' => $this->configuration_dir.'/redis.conf',
@@ -195,6 +188,9 @@ class StartRedis
         $docker_run_options = convertDockerRunToCompose($this->database->custom_docker_run_options);
         $docker_compose = generateCustomDockerRunOptionsForDatabases($docker_run_options, $docker_compose, $container_name, $this->database->destination->network);
 
+        if (! $this->database->isHealthcheckEnabled()) {
+            unset($docker_compose['services'][$container_name]['healthcheck']);
+        }
         $docker_compose = Yaml::dump($docker_compose, 10);
         $docker_compose_base64 = base64_encode($docker_compose);
         $this->commands[] = "echo '{$docker_compose_base64}' | base64 -d | tee $this->configuration_dir/docker-compose.yml > /dev/null";
@@ -205,6 +201,11 @@ class StartRedis
         if ($this->database->enable_ssl) {
             $this->commands[] = "chown -R 999:999 $this->configuration_dir/ssl/server.key $this->configuration_dir/ssl/server.crt";
         }
+        if (! is_null($this->database->redis_conf) && ! empty($this->database->redis_conf)) {
+            $this->commands[] = "chown 999:999 $this->configuration_dir/redis.conf";
+        }
+        $this->commands[] = "docker stop -t 10 $container_name 2>/dev/null || true";
+        $this->commands[] = "docker rm -f $container_name 2>/dev/null || true";
         $this->commands[] = "docker compose -f $this->configuration_dir/docker-compose.yml up -d";
         $this->commands[] = "echo 'Database started.'";
 
@@ -314,9 +315,8 @@ class StartRedis
             return;
         }
         $filename = 'redis.conf';
-        Storage::disk('local')->put("tmp/redis.conf_{$this->database->uuid}", $this->database->redis_conf);
-        $path = Storage::path("tmp/redis.conf_{$this->database->uuid}");
-        instant_scp($path, "{$this->configuration_dir}/{$filename}", $this->database->destination->server);
-        Storage::disk('local')->delete("tmp/redis.conf_{$this->database->uuid}");
+        $content = $this->database->redis_conf;
+        $content_base64 = base64_encode($content);
+        $this->commands[] = "echo '{$content_base64}' | base64 -d | tee $this->configuration_dir/{$filename} > /dev/null";
     }
 }

app/Actions/Database/StopDatabase.php

@@ -28,6 +28,13 @@ class StopDatabase
 
             $this->stopContainer($database, $database->uuid, 30);
 
+            // Reset restart tracking when database is manually stopped
+            $database->update([
+                'restart_count' => 0,
+                'last_restart_at' => null,
+                'last_restart_type' => null,
+            ]);
+
             if ($dockerCleanup) {
                 CleanupDocker::dispatch($server, false, false);
             }
@@ -49,7 +56,7 @@ class StopDatabase
     {
         $server = $database->destination->server;
         instant_remote_process(command: [
-            "docker stop --time=$timeout $containerName",
+            "docker stop -t $timeout $containerName",
             "docker rm -f $containerName",
         ], server: $server, throwError: false);
     }

app/Actions/Database/StopDatabaseProxy.php

@@ -25,7 +25,6 @@ class StopDatabaseProxy
         $server = data_get($database, 'destination.server');
         $uuid = $database->uuid;
         if ($database->getMorphClass() === \App\Models\ServiceDatabase::class) {
-            $uuid = $database->service->uuid;
             $server = data_get($database, 'service.server');
         }
         instant_remote_process(["docker rm -f {$uuid}-proxy"], $server);

app/Actions/Destination/RemoveStandaloneDockerNetwork.php

@@ -0,0 +1,16 @@
+<?php
+
+namespace App\Actions\Destination;
+
+use App\Models\StandaloneDocker;
+
+class RemoveStandaloneDockerNetwork
+{
+    public function handle(StandaloneDocker $destination): void
+    {
+        $safeNetwork = escapeshellarg($destination->network);
+
+        instant_remote_process(["docker network disconnect {$safeNetwork} coolify-proxy"], $destination->server, throwError: false);
+        instant_remote_process(["docker network rm -f {$safeNetwork}"], $destination->server);
+    }
+}

app/Actions/Docker/GetContainersStatus.php

@@ -2,19 +2,26 @@
 
 namespace App\Actions\Docker;
 
+use App\Actions\Application\StopApplication;
 use App\Actions\Database\StartDatabaseProxy;
+use App\Actions\Database\StopDatabaseProxy;
 use App\Actions\Shared\ComplexStatusCheck;
 use App\Events\ServiceChecked;
 use App\Models\ApplicationPreview;
 use App\Models\Server;
 use App\Models\ServiceDatabase;
+use App\Notifications\Application\RestartLimitReached as ApplicationRestartLimitReached;
+use App\Services\ContainerStatusAggregator;
+use App\Traits\CalculatesExcludedStatus;
 use Illuminate\Support\Arr;
 use Illuminate\Support\Collection;
+use Illuminate\Support\Facades\DB;
 use Lorisleiva\Actions\Concerns\AsAction;
 
 class GetContainersStatus
 {
     use AsAction;
+    use CalculatesExcludedStatus;
 
     public string $jobQueue = 'high';
 
@@ -26,6 +33,12 @@ class GetContainersStatus
 
     public $server;
 
+    protected ?Collection $applicationContainerStatuses;
+
+    protected ?Collection $applicationContainerRestartCounts;
+
+    protected ?Collection $serviceContainerStatuses;
+
     public function handle(Server $server, ?Collection $containers = null, ?Collection $containerReplicates = null)
     {
         $this->containers = $containers;
@@ -93,8 +106,16 @@ class GetContainersStatus
                 $labels = data_get($container, 'Config.Labels');
             }
             $containerStatus = data_get($container, 'State.Status');
-            $containerHealth = data_get($container, 'State.Health.Status', 'unhealthy');
-            $containerStatus = "$containerStatus ($containerHealth)";
+            $containerHealth = data_get($container, 'State.Health.Status');
+            if ($containerStatus === 'restarting') {
+                $healthSuffix = $containerHealth ?? 'unknown';
+                $containerStatus = "restarting:$healthSuffix";
+            } elseif ($containerStatus === 'exited') {
+                // Keep as-is, no health suffix for exited containers
+            } else {
+                $healthSuffix = $containerHealth ?? 'unknown';
+                $containerStatus = "$containerStatus:$healthSuffix";
+            }
             $labels = Arr::undot(format_docker_labels_to_json($labels));
             $applicationId = data_get($labels, 'coolify.applicationId');
             if ($applicationId) {
@@ -119,11 +140,34 @@ class GetContainersStatus
                     $application = $this->applications->where('id', $applicationId)->first();
                     if ($application) {
                         $foundApplications[] = $application->id;
-                        $statusFromDb = $application->status;
-                        if ($statusFromDb !== $containerStatus) {
-                            $application->update(['status' => $containerStatus]);
-                        } else {
-                            $application->update(['last_online_at' => now()]);
+                        // Store container status for aggregation
+                        if (! isset($this->applicationContainerStatuses)) {
+                            $this->applicationContainerStatuses = collect();
+                        }
+                        if (! $this->applicationContainerStatuses->has($applicationId)) {
+                            $this->applicationContainerStatuses->put($applicationId, collect());
+                        }
+                        $containerName = data_get($labels, 'com.docker.compose.service');
+                        // Fallback for Docker Swarm which uses different labels
+                        if (! $containerName && $this->server->isSwarm()) {
+                            $containerName = data_get($labels, 'coolify.serviceName')
+                                ?? data_get($labels, 'coolify.name')
+                                ?? data_get($labels, 'com.docker.stack.namespace');
+                        }
+                        if ($containerName) {
+                            $this->applicationContainerStatuses->get($applicationId)->put($containerName, $containerStatus);
+                        }
+
+                        // Track restart counts for applications
+                        $restartCount = data_get($container, 'RestartCount', 0);
+                        if (! isset($this->applicationContainerRestartCounts)) {
+                            $this->applicationContainerRestartCounts = collect();
+                        }
+                        if (! $this->applicationContainerRestartCounts->has($applicationId)) {
+                            $this->applicationContainerRestartCounts->put($applicationId, collect());
+                        }
+                        if ($containerName) {
+                            $this->applicationContainerRestartCounts->get($applicationId)->put($containerName, $restartCount);
                         }
                     } else {
                         // Notify user that this container should not be there.
@@ -139,21 +183,30 @@ class GetContainersStatus
                         if ($database_id) {
                             $service_db = ServiceDatabase::where('id', $database_id)->first();
                             if ($service_db) {
-                                $uuid = data_get($service_db, 'service.uuid');
-                                if ($uuid) {
-                                    $isPublic = data_get($service_db, 'is_public');
-                                    if ($isPublic) {
-                                        $foundTcpProxy = $this->containers->filter(function ($value, $key) use ($uuid) {
-                                            if ($this->server->isSwarm()) {
-                                                return data_get($value, 'Spec.Name') === "coolify-proxy_$uuid";
-                                            } else {
-                                                return data_get($value, 'Name') === "/$uuid-proxy";
-                                            }
-                                        })->first();
-                                        if (! $foundTcpProxy) {
-                                            StartDatabaseProxy::run($service_db);
-                                            // $this->server->team?->notify(new ContainerRestarted("TCP Proxy for {$service_db->service->name}", $this->server));
+                                $proxyUuid = $service_db->uuid;
+                                $isPublic = data_get($service_db, 'is_public');
+                                if ($isPublic) {
+                                    $foundTcpProxy = $this->containers->filter(function ($value, $key) use ($proxyUuid) {
+                                        if ($this->server->isSwarm()) {
+                                            return data_get($value, 'Spec.Name') === "coolify-proxy_$proxyUuid";
+                                        } else {
+                                            return data_get($value, 'Name') === "/$proxyUuid-proxy";
                                         }
+                                    })->first();
+                                    if (! $foundTcpProxy) {
+                                        StartDatabaseProxy::run($service_db);
+                                    }
+                                } else {
+                                    // Clean up orphaned proxy when is_public=false
+                                    $orphanedProxy = $this->containers->filter(function ($value, $key) use ($proxyUuid) {
+                                        if ($this->server->isSwarm()) {
+                                            return data_get($value, 'Spec.Name') === "coolify-proxy_$proxyUuid";
+                                        } else {
+                                            return data_get($value, 'Name') === "/$proxyUuid-proxy";
+                                        }
+                                    })->first();
+                                    if ($orphanedProxy) {
+                                        StopDatabaseProxy::run($service_db);
                                     }
                                 }
                             }
@@ -164,12 +217,26 @@ class GetContainersStatus
                             $isPublic = data_get($database, 'is_public');
                             $foundDatabases[] = $database->id;
                             $statusFromDb = $database->status;
+
+                            // Track restart count for databases (single-container)
+                            $restartCount = data_get($container, 'RestartCount', 0);
+                            $previousRestartCount = $database->restart_count ?? 0;
+
                             if ($statusFromDb !== $containerStatus) {
-                                $database->update(['status' => $containerStatus]);
+                                $updateData = ['status' => $containerStatus];
                             } else {
-                                $database->update(['last_online_at' => now()]);
+                                $updateData = ['last_online_at' => now()];
                             }
 
+                            // Update restart tracking if restart count increased
+                            if ($restartCount > $previousRestartCount) {
+                                $updateData['restart_count'] = $restartCount;
+                                $updateData['last_restart_at'] = now();
+                                $updateData['last_restart_type'] = 'crash';
+                            }
+
+                            $database->update($updateData);
+
                             if ($isPublic) {
                                 $foundTcpProxy = $this->containers->filter(function ($value, $key) use ($uuid) {
                                     if ($this->server->isSwarm()) {
@@ -180,7 +247,18 @@ class GetContainersStatus
                                 })->first();
                                 if (! $foundTcpProxy) {
                                     StartDatabaseProxy::run($database);
-                                    // $this->server->team?->notify(new ContainerRestarted("TCP Proxy for database", $this->server));
+                                }
+                            } else {
+                                // Clean up orphaned proxy when is_public=false
+                                $orphanedProxy = $this->containers->filter(function ($value, $key) use ($uuid) {
+                                    if ($this->server->isSwarm()) {
+                                        return data_get($value, 'Spec.Name') === "coolify-proxy_$uuid";
+                                    } else {
+                                        return data_get($value, 'Name') === "/$uuid-proxy";
+                                    }
+                                })->first();
+                                if ($orphanedProxy) {
+                                    StopDatabaseProxy::run($database);
                                 }
                             }
                         } else {
@@ -196,23 +274,34 @@ class GetContainersStatus
             if ($serviceLabelId) {
                 $subType = data_get($labels, 'coolify.service.subType');
                 $subId = data_get($labels, 'coolify.service.subId');
-                $service = $services->where('id', $serviceLabelId)->first();
-                if (! $service) {
+                $parentService = $services->where('id', $serviceLabelId)->first();
+                if (! $parentService) {
                     continue;
                 }
+
+                // Store container status for aggregation
+                if (! isset($this->serviceContainerStatuses)) {
+                    $this->serviceContainerStatuses = collect();
+                }
+
+                $key = $serviceLabelId.':'.$subType.':'.$subId;
+                if (! $this->serviceContainerStatuses->has($key)) {
+                    $this->serviceContainerStatuses->put($key, collect());
+                }
+
+                $containerName = data_get($labels, 'com.docker.compose.service');
+                if ($containerName) {
+                    $this->serviceContainerStatuses->get($key)->put($containerName, $containerStatus);
+                }
+
+                // Mark service as found
                 if ($subType === 'application') {
-                    $service = $service->applications()->where('id', $subId)->first();
+                    $service = $parentService->applications()->where('id', $subId)->first();
                 } else {
-                    $service = $service->databases()->where('id', $subId)->first();
+                    $service = $parentService->databases()->where('id', $subId)->first();
                 }
                 if ($service) {
                     $foundServices[] = "$service->id-$service->name";
-                    $statusFromDb = $service->status;
-                    if ($statusFromDb !== $containerStatus) {
-                        $service->update(['status' => $containerStatus]);
-                    } else {
-                        $service->update(['last_online_at' => now()]);
-                    }
                 }
             }
         }
@@ -240,6 +329,12 @@ class GetContainersStatus
             if (str($exitedService->status)->startsWith('exited')) {
                 continue;
             }
+
+            // Only protection: If no containers at all, Docker query might have failed
+            if ($this->containers->isEmpty()) {
+                continue;
+            }
+
             $name = data_get($exitedService, 'name');
             $fqdn = data_get($exitedService, 'fqdn');
             if ($name) {
@@ -280,7 +375,24 @@ class GetContainersStatus
                 continue;
             }
 
-            $application->update(['status' => 'exited']);
+            // If container was recently restarting (crash loop), keep it as degraded for a grace period
+            // This prevents false "exited" status during the brief moment between container removal and recreation
+            $recentlyRestarted = $application->restart_count > 0 &&
+                                 $application->last_restart_at &&
+                                 $application->last_restart_at->greaterThan(now()->subSeconds(30));
+
+            if ($recentlyRestarted) {
+                // Keep it as degraded if it was recently in a crash loop
+                $application->update(['status' => 'degraded:unhealthy']);
+            } else {
+                // Reset restart count when application exits completely
+                $application->update([
+                    'status' => 'exited',
+                    'restart_count' => 0,
+                    'last_restart_at' => null,
+                    'last_restart_type' => null,
+                ]);
+            }
         }
         $notRunningApplicationPreviews = $previews->pluck('id')->diff($foundApplicationPreviews);
         foreach ($notRunningApplicationPreviews as $previewId) {
@@ -302,7 +414,24 @@ class GetContainersStatus
             if (str($database->status)->startsWith('exited')) {
                 continue;
             }
-            $database->update(['status' => 'exited']);
+
+            // Only protection: If no containers at all, Docker query might have failed
+            if ($this->containers->isEmpty()) {
+                continue;
+            }
+
+            // Reset restart tracking when database exits completely
+            $database->update([
+                'status' => 'exited',
+                'restart_count' => 0,
+                'last_restart_at' => null,
+                'last_restart_type' => null,
+            ]);
+
+            // Stop proxy if database was public
+            if ($database->is_public) {
+                StopDatabaseProxy::run($database);
+            }
 
             $name = data_get($database, 'name');
             $fqdn = data_get($database, 'fqdn');
@@ -320,6 +449,157 @@ class GetContainersStatus
             }
             // $this->server->team?->notify(new ContainerStopped($containerName, $this->server, $url));
         }
+
+        // Aggregate multi-container application statuses
+        if (isset($this->applicationContainerStatuses) && $this->applicationContainerStatuses->isNotEmpty()) {
+            foreach ($this->applicationContainerStatuses as $applicationId => $containerStatuses) {
+                $application = $this->applications->where('id', $applicationId)->first();
+                if (! $application) {
+                    continue;
+                }
+
+                // Track restart counts first
+                $maxRestartCount = 0;
+                if (isset($this->applicationContainerRestartCounts) && $this->applicationContainerRestartCounts->has($applicationId)) {
+                    $containerRestartCounts = $this->applicationContainerRestartCounts->get($applicationId);
+                    $maxRestartCount = $containerRestartCounts->max() ?? 0;
+                }
+
+                // Wrap all database updates in a transaction to ensure consistency
+                $restartLimitReached = false;
+
+                DB::transaction(function () use ($application, $maxRestartCount, $containerStatuses, &$restartLimitReached) {
+                    $previousRestartCount = $application->restart_count ?? 0;
+
+                    if ($maxRestartCount > $previousRestartCount) {
+                        // Restart count increased - this is a crash restart
+                        $application->update([
+                            'restart_count' => $maxRestartCount,
+                            'last_restart_at' => now(),
+                            'last_restart_type' => 'crash',
+                        ]);
+
+                        // Check if restart limit has been reached
+                        $maxAllowedRestarts = $application->max_restart_count ?? 0;
+                        if ($maxAllowedRestarts > 0 && $maxRestartCount >= $maxAllowedRestarts && $previousRestartCount < $maxAllowedRestarts) {
+                            $restartLimitReached = true;
+                        }
+                    }
+
+                    // Aggregate status after tracking restart counts
+                    $aggregatedStatus = $this->aggregateApplicationStatus($application, $containerStatuses, $maxRestartCount);
+                    if ($aggregatedStatus) {
+                        $statusFromDb = $application->status;
+                        if ($statusFromDb !== $aggregatedStatus) {
+                            $application->update(['status' => $aggregatedStatus]);
+                        } else {
+                            $application->update(['last_online_at' => now()]);
+                        }
+                    }
+                });
+
+                if ($restartLimitReached) {
+                    $application->refresh();
+                    StopApplication::dispatch($application, false, true, false);
+                    $application->environment->project->team?->notify(new ApplicationRestartLimitReached($application));
+                }
+            }
+        }
+
+        // Aggregate multi-container service statuses
+        $this->aggregateServiceContainerStatuses($services);
+
         ServiceChecked::dispatch($this->server->team->id);
     }
+
+    private function aggregateApplicationStatus($application, Collection $containerStatuses, int $maxRestartCount = 0): ?string
+    {
+        // Parse docker compose to check for excluded containers
+        $dockerComposeRaw = data_get($application, 'docker_compose_raw');
+        $excludedContainers = $this->getExcludedContainersFromDockerCompose($dockerComposeRaw);
+
+        // Filter out excluded containers
+        $relevantStatuses = $containerStatuses->filter(function ($status, $containerName) use ($excludedContainers) {
+            return ! $excludedContainers->contains($containerName);
+        });
+
+        // If all containers are excluded, calculate status from excluded containers
+        if ($relevantStatuses->isEmpty()) {
+            return $this->calculateExcludedStatusFromStrings($containerStatuses);
+        }
+
+        // Use ContainerStatusAggregator service for state machine logic
+        // Use preserveRestarting: true so applications show "Restarting" instead of "Degraded"
+        $aggregator = new ContainerStatusAggregator;
+
+        return $aggregator->aggregateFromStrings($relevantStatuses, $maxRestartCount, preserveRestarting: true);
+    }
+
+    private function aggregateServiceContainerStatuses($services)
+    {
+        if (! isset($this->serviceContainerStatuses) || $this->serviceContainerStatuses->isEmpty()) {
+            return;
+        }
+
+        foreach ($this->serviceContainerStatuses as $key => $containerStatuses) {
+            // Parse key: serviceId:subType:subId
+            [$serviceId, $subType, $subId] = explode(':', $key);
+
+            $service = $services->where('id', $serviceId)->first();
+            if (! $service) {
+                continue;
+            }
+
+            // Get the service sub-resource (ServiceApplication or ServiceDatabase)
+            $subResource = null;
+            if ($subType === 'application') {
+                $subResource = $service->applications()->where('id', $subId)->first();
+            } elseif ($subType === 'database') {
+                $subResource = $service->databases()->where('id', $subId)->first();
+            }
+
+            if (! $subResource) {
+                continue;
+            }
+
+            // Parse docker compose from service to check for excluded containers
+            $dockerComposeRaw = data_get($service, 'docker_compose_raw');
+            $excludedContainers = $this->getExcludedContainersFromDockerCompose($dockerComposeRaw);
+
+            // Filter out excluded containers
+            $relevantStatuses = $containerStatuses->filter(function ($status, $containerName) use ($excludedContainers) {
+                return ! $excludedContainers->contains($containerName);
+            });
+
+            // If all containers are excluded, calculate status from excluded containers
+            if ($relevantStatuses->isEmpty()) {
+                $aggregatedStatus = $this->calculateExcludedStatusFromStrings($containerStatuses);
+                if ($aggregatedStatus) {
+                    $statusFromDb = $subResource->status;
+                    if ($statusFromDb !== $aggregatedStatus) {
+                        $subResource->update(['status' => $aggregatedStatus]);
+                    } else {
+                        $subResource->update(['last_online_at' => now()]);
+                    }
+                }
+
+                continue;
+            }
+
+            // Use ContainerStatusAggregator service for state machine logic
+            // Use preserveRestarting: true so individual sub-resources show "Restarting" instead of "Degraded"
+            $aggregator = new ContainerStatusAggregator;
+            $aggregatedStatus = $aggregator->aggregateFromStrings($relevantStatuses, preserveRestarting: true);
+
+            // Update service sub-resource status with aggregated result
+            if ($aggregatedStatus) {
+                $statusFromDb = $subResource->status;
+                if ($statusFromDb !== $aggregatedStatus) {
+                    $subResource->update(['status' => $aggregatedStatus]);
+                } else {
+                    $subResource->update(['last_online_at' => now()]);
+                }
+            }
+        }
+    }
 }

app/Actions/Fortify/CreateNewUser.php

@@ -2,6 +2,7 @@
 
 namespace App\Actions\Fortify;
 
+use App\Models\Team;
 use App\Models\User;
 use Illuminate\Support\Facades\Hash;
 use Illuminate\Support\Facades\Validator;
@@ -37,13 +38,17 @@ class CreateNewUser implements CreatesNewUsers
         if (User::count() == 0) {
             // If this is the first user, make them the root user
             // Team is already created in the database/seeders/ProductionSeeder.php
-            $user = User::create([
+            $user = (new User)->forceFill([
                 'id' => 0,
                 'name' => $input['name'],
-                'email' => strtolower($input['email']),
+                'email' => $input['email'],
                 'password' => Hash::make($input['password']),
             ]);
-            $team = $user->teams()->first();
+            $user->save();
+            $team = $user->teams()->first() ?? Team::find(0);
+            if ($team !== null && ! $user->teams()->where('team_id', $team->id)->exists()) {
+                $user->teams()->attach($team, ['role' => 'owner']);
+            }
 
             // Disable registration after first user is created
             $settings = instanceSettings();
@@ -52,7 +57,7 @@ class CreateNewUser implements CreatesNewUsers
         } else {
             $user = User::create([
                 'name' => $input['name'],
-                'email' => strtolower($input['email']),
+                'email' => $input['email'],
                 'password' => Hash::make($input['password']),
             ]);
             $team = $user->teams()->first();

app/Actions/Fortify/ResetUserPassword.php

@@ -21,7 +21,7 @@ class ResetUserPassword implements ResetsUserPasswords
             'password' => ['required', Password::defaults(), 'confirmed'],
         ])->validate();
 
-        $user->forceFill([
+        $user->fill([
             'password' => Hash::make($input['password']),
         ])->save();
         $user->deleteAllSessions();

app/Actions/Fortify/UpdateUserPassword.php

@@ -24,7 +24,7 @@ class UpdateUserPassword implements UpdatesUserPasswords
             'current_password.current_password' => __('The provided password does not match your current password.'),
         ])->validateWithBag('updatePassword');
 
-        $user->forceFill([
+        $user->fill([
             'password' => Hash::make($input['password']),
         ])->save();
     }

app/Actions/Fortify/UpdateUserProfileInformation.php

@@ -35,7 +35,7 @@ class UpdateUserProfileInformation implements UpdatesUserProfileInformation
         ) {
             $this->updateVerifiedUser($user, $input);
         } else {
-            $user->forceFill([
+            $user->fill([
                 'name' => $input['name'],
                 'email' => $input['email'],
             ])->save();
@@ -49,7 +49,7 @@ class UpdateUserProfileInformation implements UpdatesUserProfileInformation
      */
     protected function updateVerifiedUser(User $user, array $input): void
     {
-        $user->forceFill([
+        $user->fill([
             'name' => $input['name'],
             'email' => $input['email'],
             'email_verified_at' => null,

app/Actions/Proxy/CheckConfiguration.php

@@ -1,36 +0,0 @@
-<?php
-
-namespace App\Actions\Proxy;
-
-use App\Models\Server;
-use App\Services\ProxyDashboardCacheService;
-use Lorisleiva\Actions\Concerns\AsAction;
-
-class CheckConfiguration
-{
-    use AsAction;
-
-    public function handle(Server $server, bool $reset = false)
-    {
-        $proxyType = $server->proxyType();
-        if ($proxyType === 'NONE') {
-            return 'OK';
-        }
-        $proxy_path = $server->proxyPath();
-        $payload = [
-            "mkdir -p $proxy_path",
-            "cat $proxy_path/docker-compose.yml",
-        ];
-        $proxy_configuration = instant_remote_process($payload, $server, false);
-        if ($reset || ! $proxy_configuration || is_null($proxy_configuration)) {
-            $proxy_configuration = str(generate_default_proxy_configuration($server))->trim()->value();
-        }
-        if (! $proxy_configuration || is_null($proxy_configuration)) {
-            throw new \Exception('Could not generate proxy configuration');
-        }
-
-        ProxyDashboardCacheService::isTraefikDashboardAvailableFromConfiguration($server, $proxy_configuration);
-
-        return $proxy_configuration;
-    }
-}

app/Actions/Proxy/CheckProxy.php

@@ -70,7 +70,7 @@ class CheckProxy
 
             try {
                 if ($server->proxyType() !== ProxyTypes::NONE->value) {
-                    $proxyCompose = CheckConfiguration::run($server);
+                    $proxyCompose = GetProxyConfiguration::run($server);
                     if (isset($proxyCompose)) {
                         $yaml = Yaml::parse($proxyCompose);
                         $configPorts = [];

app/Actions/Proxy/GetProxyConfiguration.php

@@ -0,0 +1,119 @@
+<?php
+
+namespace App\Actions\Proxy;
+
+use App\Enums\ProxyTypes;
+use App\Models\Server;
+use App\Services\ProxyDashboardCacheService;
+use Illuminate\Support\Facades\Log;
+use Lorisleiva\Actions\Concerns\AsAction;
+use Symfony\Component\Yaml\Yaml;
+
+class GetProxyConfiguration
+{
+    use AsAction;
+
+    public function handle(Server $server, bool $forceRegenerate = false): string
+    {
+        $proxyType = $server->proxyType();
+        if ($proxyType === 'NONE') {
+            return 'OK';
+        }
+
+        $proxy_configuration = null;
+
+        if (! $forceRegenerate) {
+            // Primary source: database
+            $proxy_configuration = $server->proxy->get('last_saved_proxy_configuration');
+
+            // Validate stored config matches current proxy type
+            if (! empty(trim($proxy_configuration ?? ''))) {
+                if (! $this->configMatchesProxyType($proxyType, $proxy_configuration)) {
+                    Log::warning('Stored proxy config does not match current proxy type, will regenerate', [
+                        'server_id' => $server->id,
+                        'proxy_type' => $proxyType,
+                    ]);
+                    $proxy_configuration = null;
+                }
+            }
+
+            // Backfill: existing servers may not have DB config yet — read from disk once
+            if (empty(trim($proxy_configuration ?? ''))) {
+                $proxy_configuration = $this->backfillFromDisk($server);
+            }
+        }
+
+        // Generate default configuration as last resort
+        if ($forceRegenerate || empty(trim($proxy_configuration ?? ''))) {
+            $custom_commands = [];
+            if (! empty(trim($proxy_configuration ?? ''))) {
+                $custom_commands = extractCustomProxyCommands($server, $proxy_configuration);
+            }
+
+            Log::warning('Proxy configuration regenerated to defaults', [
+                'server_id' => $server->id,
+                'server_name' => $server->name,
+                'reason' => $forceRegenerate ? 'force_regenerate' : 'config_not_found',
+            ]);
+
+            $proxy_configuration = str(generateDefaultProxyConfiguration($server, $custom_commands))->trim()->value();
+        }
+
+        if (empty($proxy_configuration)) {
+            throw new \Exception('Could not get or generate proxy configuration');
+        }
+
+        ProxyDashboardCacheService::isTraefikDashboardAvailableFromConfiguration($server, $proxy_configuration);
+
+        return $proxy_configuration;
+    }
+
+    /**
+     * Check that the stored docker-compose YAML contains the expected service
+     * for the server's current proxy type. Returns false if the config belongs
+     * to a different proxy type (e.g. Traefik config on a CADDY server).
+     */
+    private function configMatchesProxyType(string $proxyType, string $configuration): bool
+    {
+        try {
+            $yaml = Yaml::parse($configuration);
+            $services = data_get($yaml, 'services', []);
+
+            return match ($proxyType) {
+                ProxyTypes::TRAEFIK->value => isset($services['traefik']),
+                ProxyTypes::CADDY->value => isset($services['caddy']),
+                ProxyTypes::NGINX->value => isset($services['nginx']),
+                default => true,
+            };
+        } catch (\Throwable $e) {
+            // If YAML is unparseable, don't block — let the existing flow handle it
+            return true;
+        }
+    }
+
+    /**
+     * Backfill: read config from disk for servers that predate DB storage.
+     * Stores the result in the database so future reads skip SSH entirely.
+     */
+    private function backfillFromDisk(Server $server): ?string
+    {
+        $proxy_path = $server->proxyPath();
+        $result = instant_remote_process([
+            "mkdir -p $proxy_path",
+            "cat $proxy_path/docker-compose.yml 2>/dev/null",
+        ], $server, false);
+
+        if (! empty(trim($result ?? ''))) {
+            $server->proxy->last_saved_proxy_configuration = $result;
+            $server->save();
+
+            Log::info('Proxy config backfilled to database from disk', [
+                'server_id' => $server->id,
+            ]);
+
+            return $result;
+        }
+
+        return null;
+    }
+}

app/Actions/Proxy/SaveConfiguration.php

@@ -1,28 +0,0 @@
-<?php
-
-namespace App\Actions\Proxy;
-
-use App\Models\Server;
-use Lorisleiva\Actions\Concerns\AsAction;
-
-class SaveConfiguration
-{
-    use AsAction;
-
-    public function handle(Server $server, ?string $proxy_settings = null)
-    {
-        if (is_null($proxy_settings)) {
-            $proxy_settings = CheckConfiguration::run($server, true);
-        }
-        $proxy_path = $server->proxyPath();
-        $docker_compose_yml_base64 = base64_encode($proxy_settings);
-
-        $server->proxy->last_saved_settings = str($docker_compose_yml_base64)->pipe('md5')->value;
-        $server->save();
-
-        return instant_remote_process([
-            "mkdir -p $proxy_path",
-            "echo '$docker_compose_yml_base64' | base64 -d | tee $proxy_path/docker-compose.yml > /dev/null",
-        ], $server);
-    }
-}

app/Actions/Proxy/SaveProxyConfiguration.php

@@ -0,0 +1,49 @@
+<?php
+
+namespace App\Actions\Proxy;
+
+use App\Models\Server;
+use Lorisleiva\Actions\Concerns\AsAction;
+
+class SaveProxyConfiguration
+{
+    use AsAction;
+
+    private const MAX_BACKUPS = 10;
+
+    public function handle(Server $server, string $configuration): void
+    {
+        $proxy_path = $server->proxyPath();
+        $docker_compose_yml_base64 = base64_encode($configuration);
+        $new_hash = str($docker_compose_yml_base64)->pipe('md5')->value;
+
+        // Only create a backup if the configuration actually changed
+        $old_hash = $server->proxy->get('last_saved_settings');
+        $config_changed = $old_hash && $old_hash !== $new_hash;
+
+        // Update the saved settings hash and store full config as database backup
+        $server->proxy->last_saved_settings = $new_hash;
+        $server->proxy->last_saved_proxy_configuration = $configuration;
+        $server->save();
+
+        $backup_path = "$proxy_path/backups";
+
+        // Transfer the configuration file to the server, with backup if changed
+        $commands = ["mkdir -p $proxy_path"];
+
+        if ($config_changed) {
+            $short_hash = substr($old_hash, 0, 8);
+            $timestamp = now()->format('Y-m-d_H-i-s');
+            $backup_file = "docker-compose.{$timestamp}.{$short_hash}.yml";
+            $commands[] = "mkdir -p $backup_path";
+            // Skip backup if a file with the same hash already exists (identical content)
+            $commands[] = "ls $backup_path/docker-compose.*.$short_hash.yml 1>/dev/null 2>&1 || cp -f $proxy_path/docker-compose.yml $backup_path/$backup_file 2>/dev/null || true";
+            // Prune old backups, keep only the most recent ones
+            $commands[] = 'cd '.$backup_path.' && ls -1t docker-compose.*.yml 2>/dev/null | tail -n +'.((int) self::MAX_BACKUPS + 1).' | xargs rm -f 2>/dev/null || true';
+        }
+
+        $commands[] = "echo '$docker_compose_yml_base64' | base64 -d | tee $proxy_path/docker-compose.yml > /dev/null";
+
+        instant_remote_process($commands, $server);
+    }
+}

app/Actions/Proxy/StartProxy.php

@@ -13,19 +13,27 @@ class StartProxy
 {
     use AsAction;
 
-    public function handle(Server $server, bool $async = true, bool $force = false): string|Activity
+    public function handle(Server $server, bool $async = true, bool $force = false, bool $restarting = false): string|Activity
     {
         $proxyType = $server->proxyType();
         if ((is_null($proxyType) || $proxyType === 'NONE' || $server->proxy->force_stop || $server->isBuildServer()) && $force === false) {
             return 'OK';
         }
+        $server->proxy->set('status', 'starting');
+        $server->save();
+        $server->refresh();
+
+        if (! $restarting) {
+            ProxyStatusChangedUI::dispatch($server->team_id);
+        }
+
         $commands = collect([]);
         $proxy_path = $server->proxyPath();
-        $configuration = CheckConfiguration::run($server);
+        $configuration = GetProxyConfiguration::run($server);
         if (! $configuration) {
             throw new \Exception('Configuration is not synced');
         }
-        SaveConfiguration::run($server, $configuration);
+        SaveProxyConfiguration::run($server, $configuration);
         $docker_compose_yml_base64 = base64_encode($configuration);
         $server->proxy->last_applied_settings = str($docker_compose_yml_base64)->pipe('md5')->value();
         $server->save();
@@ -55,23 +63,34 @@ class StartProxy
                 'docker compose pull',
                 'if docker ps -a --format "{{.Names}}" | grep -q "^coolify-proxy$"; then',
                 "    echo 'Stopping and removing existing coolify-proxy.'",
-                '    docker rm -f coolify-proxy || true',
+                '    docker stop coolify-proxy 2>/dev/null || true',
+                '    docker rm -f coolify-proxy 2>/dev/null || true',
+                '    # Wait for container to be fully removed',
+                '    for i in {1..10}; do',
+                '        if ! docker ps -a --format "{{.Names}}" | grep -q "^coolify-proxy$"; then',
+                '            break',
+                '        fi',
+                '        echo "Waiting for coolify-proxy to be removed... ($i/10)"',
+                '        sleep 1',
+                '    done',
                 "    echo 'Successfully stopped and removed existing coolify-proxy.'",
                 'fi',
+            ]);
+            // Ensure required networks exist BEFORE docker compose up (networks are declared as external)
+            $commands = $commands->merge(ensureProxyNetworksExist($server));
+            $commands = $commands->merge([
                 "echo 'Starting coolify-proxy.'",
                 'docker compose up -d --wait --remove-orphans',
                 "echo 'Successfully started coolify-proxy.'",
             ]);
             $commands = $commands->merge(connectProxyToNetworks($server));
         }
-        $server->proxy->set('status', 'starting');
-        $server->save();
-        ProxyStatusChangedUI::dispatch($server->team_id);
 
         if ($async) {
             return remote_process($commands, $server, callEventOnFinish: 'ProxyStatusChanged', callEventData: $server->id);
         } else {
             instant_remote_process($commands, $server);
+
             $server->proxy->set('type', $proxyType);
             $server->save();
             ProxyStatusChanged::dispatch($server->id);

app/Actions/Proxy/StopProxy.php

@@ -12,17 +12,27 @@ class StopProxy
 {
     use AsAction;
 
-    public function handle(Server $server, bool $forceStop = true, int $timeout = 30)
+    public function handle(Server $server, bool $forceStop = true, int $timeout = 30, bool $restarting = false)
     {
         try {
             $containerName = $server->isSwarm() ? 'coolify-proxy_traefik' : 'coolify-proxy';
             $server->proxy->status = 'stopping';
             $server->save();
-            ProxyStatusChangedUI::dispatch($server->team_id);
+
+            if (! $restarting) {
+                ProxyStatusChangedUI::dispatch($server->team_id);
+            }
 
             instant_remote_process(command: [
-                "docker stop --time=$timeout $containerName",
-                "docker rm -f $containerName",
+                "docker stop -t=$timeout $containerName 2>/dev/null || true",
+                "docker rm -f $containerName 2>/dev/null || true",
+                '# Wait for container to be fully removed',
+                'for i in {1..10}; do',
+                "    if ! docker ps -a --format \"{{.Names}}\" | grep -q \"^$containerName$\"; then",
+                '        break',
+                '    fi',
+                '    sleep 1',
+                'done',
             ], server: $server, throwError: false);
 
             $server->proxy->force_stop = $forceStop;
@@ -32,7 +42,10 @@ class StopProxy
             return handleError($e);
         } finally {
             ProxyDashboardCacheService::clearCache($server);
-            ProxyStatusChanged::dispatch($server->id);
+
+            if (! $restarting) {
+                ProxyStatusChanged::dispatch($server->id);
+            }
         }
     }
 }

app/Actions/Server/CheckUpdates.php

@@ -13,6 +13,9 @@ class CheckUpdates
 
     public function handle(Server $server)
     {
+        $osId = 'unknown';
+        $packageManager = null;
+
         try {
             if ($server->serverStatus() === false) {
                 return [
@@ -93,6 +96,16 @@ class CheckUpdates
                     $out['osId'] = $osId;
                     $out['package_manager'] = $packageManager;
 
+                    return $out;
+                case 'pacman':
+                    // Sync database first, then check for updates
+                    // Using -Sy to refresh package database before querying available updates
+                    instant_remote_process(['pacman -Sy'], $server);
+                    $output = instant_remote_process(['pacman -Qu 2>/dev/null'], $server);
+                    $out = $this->parsePacmanOutput($output);
+                    $out['osId'] = $osId;
+                    $out['package_manager'] = $packageManager;
+
                     return $out;
                 default:
                     return [
@@ -102,7 +115,6 @@ class CheckUpdates
                     ];
             }
         } catch (\Throwable $e) {
-            ray('Error:', $e->getMessage());
 
             return [
                 'osId' => $osId,
@@ -220,4 +232,45 @@ class CheckUpdates
             'updates' => $updates,
         ];
     }
+
+    private function parsePacmanOutput(string $output): array
+    {
+        $updates = [];
+        $unparsedLines = [];
+        $lines = explode("\n", $output);
+
+        foreach ($lines as $line) {
+            if (empty($line)) {
+                continue;
+            }
+            // Format: package current_version -> new_version
+            if (preg_match('/^(\S+)\s+(\S+)\s+->\s+(\S+)$/', $line, $matches)) {
+                $updates[] = [
+                    'package' => $matches[1],
+                    'current_version' => $matches[2],
+                    'new_version' => $matches[3],
+                    'architecture' => 'unknown',
+                    'repository' => 'unknown',
+                ];
+            } else {
+                // Log unmatched lines for debugging purposes
+                $unparsedLines[] = $line;
+            }
+        }
+
+        $result = [
+            'total_updates' => count($updates),
+            'updates' => $updates,
+        ];
+
+        // Include unparsed lines in the result for debugging if any exist
+        if (! empty($unparsedLines)) {
+            $result['unparsed_lines'] = $unparsedLines;
+            \Illuminate\Support\Facades\Log::debug('Pacman output contained unparsed lines', [
+                'unparsed_lines' => $unparsedLines,
+            ]);
+        }
+
+        return $result;
+    }
 }

app/Actions/Server/CleanupDocker.php

@@ -13,23 +13,45 @@ class CleanupDocker
 
     public function handle(Server $server, bool $deleteUnusedVolumes = false, bool $deleteUnusedNetworks = false)
     {
-        $settings = instanceSettings();
         $realtimeImage = config('constants.coolify.realtime_image');
         $realtimeImageVersion = config('constants.coolify.realtime_version');
         $realtimeImageWithVersion = "$realtimeImage:$realtimeImageVersion";
         $realtimeImageWithoutPrefix = 'coollabsio/coolify-realtime';
         $realtimeImageWithoutPrefixVersion = "coollabsio/coolify-realtime:$realtimeImageVersion";
 
-        $helperImageVersion = data_get($settings, 'helper_version');
-        $helperImage = config('constants.coolify.helper_image');
+        $helperImageVersion = getHelperVersion();
+        $helperImage = coolifyHelperImage();
         $helperImageWithVersion = "$helperImage:$helperImageVersion";
         $helperImageWithoutPrefix = 'coollabsio/coolify-helper';
         $helperImageWithoutPrefixVersion = "coollabsio/coolify-helper:$helperImageVersion";
 
+        $cleanupLog = [];
+
+        // Get all application image repositories to exclude from prune
+        $applications = $server->applications();
+        $applicationImageRepos = collect($applications)->map(function ($app) {
+            return $app->docker_registry_image_name ?? $app->uuid;
+        })->unique()->values();
+
+        // Clean up old application images while preserving N most recent for rollback
+        $applicationCleanupLog = $this->cleanupApplicationImages($server, $applications);
+        $cleanupLog = array_merge($cleanupLog, $applicationCleanupLog);
+
+        // Build image prune command that excludes application images and current Coolify infrastructure images
+        // This ensures we clean up non-Coolify images while preserving rollback images and current helper/realtime images
+        // Note: Only the current version is protected; old versions will be cleaned up by explicit commands below
+        // We pass the version strings so all registry variants are protected (ghcr.io, docker.io, no prefix)
+        $imagePruneCmd = $this->buildImagePruneCommand(
+            $applicationImageRepos,
+            $helperImageVersion,
+            $realtimeImageVersion
+        );
+
         $commands = [
-            'docker container prune -f --filter "label=coolify.managed=true" --filter "label!=coolify.proxy=true"',
-            'docker image prune -af --filter "label!=coolify.managed=true"',
+            'docker container prune -f --filter "label=coolify.managed=true" --filter "label!=coolify.proxy=true" --filter "label!=coolify.type=database" --filter "label!=coolify.type=application" --filter "label!=coolify.type=service"',
+            $imagePruneCmd,
             'docker builder prune -af',
+            "docker run --rm -v \$HOME/.docker/buildx:/root/.docker/buildx -v /var/run/docker.sock:/var/run/docker.sock {$helperImageWithVersion} docker buildx prune --builder coolify-railpack -af 2>/dev/null || true",
             "docker images --filter before=$helperImageWithVersion --filter reference=$helperImage | grep $helperImage | awk '{print $3}' | xargs -r docker rmi -f",
             "docker images --filter before=$realtimeImageWithVersion --filter reference=$realtimeImage | grep $realtimeImage | awk '{print $3}' | xargs -r docker rmi -f",
             "docker images --filter before=$helperImageWithoutPrefixVersion --filter reference=$helperImageWithoutPrefix | grep $helperImageWithoutPrefix | awk '{print $3}' | xargs -r docker rmi -f",
@@ -44,7 +66,6 @@ class CleanupDocker
             $commands[] = 'docker network prune -f';
         }
 
-        $cleanupLog = [];
         foreach ($commands as $command) {
             $commandOutput = instant_remote_process([$command], $server, false);
             if ($commandOutput !== null) {
@@ -57,4 +78,161 @@ class CleanupDocker
 
         return $cleanupLog;
     }
+
+    /**
+     * Build a docker image prune command that excludes application image repositories.
+     *
+     * Since docker image prune doesn't support excluding by repository name directly,
+     * we use a shell script approach to delete unused images while preserving application images.
+     */
+    private function buildImagePruneCommand(
+        $applicationImageRepos,
+        string $helperImageVersion,
+        string $realtimeImageVersion
+    ): string {
+        // Step 1: Always prune dangling images (untagged)
+        $commands = ['docker image prune -f'];
+
+        // Build grep pattern to exclude application image repositories (matches repo:tag and repo_service:tag)
+        $appExcludePatterns = $applicationImageRepos->map(function ($repo) {
+            // Escape special characters for grep extended regex (ERE)
+            // ERE special chars: . \ + * ? [ ^ ] $ ( ) { } |
+            return preg_replace('/([.\\\\+*?\[\]^$(){}|])/', '\\\\$1', $repo);
+        })->implode('|');
+
+        // Build grep pattern to exclude Coolify infrastructure images (current version only)
+        // This pattern matches the image name regardless of registry prefix:
+        // - ghcr.io/coollabsio/coolify-helper:1.0.12
+        // - docker.io/coollabsio/coolify-helper:1.0.12
+        // - coollabsio/coolify-helper:1.0.12
+        // Pattern: (^|/)coollabsio/coolify-(helper|realtime):VERSION$
+        $escapedHelperVersion = preg_replace('/([.\\\\+*?\[\]^$(){}|])/', '\\\\$1', $helperImageVersion);
+        $escapedRealtimeVersion = preg_replace('/([.\\\\+*?\[\]^$(){}|])/', '\\\\$1', $realtimeImageVersion);
+        $infraExcludePattern = "(^|/)coollabsio/coolify-helper:{$escapedHelperVersion}$|(^|/)coollabsio/coolify-realtime:{$escapedRealtimeVersion}$";
+
+        // Delete unused images that:
+        // - Are not application images (don't match app repos)
+        // - Are not current Coolify infrastructure images (any registry)
+        // - Don't have coolify.managed=true label
+        // Images in use by containers will fail silently with docker rmi
+        // Pattern matches both uuid:tag and uuid_servicename:tag (Docker Compose with build)
+        $grepCommands = "grep -v '<none>'";
+
+        // Add application repo exclusion if there are applications
+        if ($applicationImageRepos->isNotEmpty()) {
+            $grepCommands .= " | grep -v -E '^({$appExcludePatterns})[_:].+'";
+        }
+
+        // Add infrastructure image exclusion (matches any registry prefix)
+        $grepCommands .= " | grep -v -E '{$infraExcludePattern}'";
+
+        $commands[] = "docker images --format '{{.Repository}}:{{.Tag}}' | ".
+            $grepCommands.' | '.
+            "xargs -r -I {} sh -c 'docker inspect --format \"{{{{index .Config.Labels \\\"coolify.managed\\\"}}}}\" \"{}\" 2>/dev/null | grep -q true || docker rmi \"{}\" 2>/dev/null' || true";
+
+        return implode(' && ', $commands);
+    }
+
+    private function cleanupApplicationImages(Server $server, $applications = null): array
+    {
+        $cleanupLog = [];
+
+        if ($applications === null) {
+            $applications = $server->applications();
+        }
+
+        $disableRetention = $server->settings->disable_application_image_retention ?? false;
+
+        foreach ($applications as $application) {
+            $imagesToKeep = $disableRetention ? 0 : ($application->settings->docker_images_to_keep ?? 2);
+            $imageRepository = $application->docker_registry_image_name ?? $application->uuid;
+
+            // Get the currently running image tag
+            $currentTagCommand = "docker inspect --format='{{.Config.Image}}' {$application->uuid} 2>/dev/null | grep -oP '(?<=:)[^:]+$' || true";
+            $currentTag = instant_remote_process([$currentTagCommand], $server, false);
+            $currentTag = trim($currentTag ?? '');
+
+            // List all images for this application with their creation timestamps
+            // Use wildcard to match both uuid:tag and uuid_servicename:tag (Docker Compose with build)
+            $listCommand = "docker images --format '{{.Repository}}:{{.Tag}}#{{.CreatedAt}}' --filter reference='{$imageRepository}*' 2>/dev/null || true";
+            $output = instant_remote_process([$listCommand], $server, false);
+
+            if (empty($output)) {
+                continue;
+            }
+
+            $images = collect(explode("\n", trim($output)))
+                ->filter()
+                ->map(function ($line) {
+                    $parts = explode('#', $line);
+                    $imageRef = $parts[0] ?? '';
+                    $tagParts = explode(':', $imageRef);
+
+                    return [
+                        'repository' => $tagParts[0] ?? '',
+                        'tag' => $tagParts[1] ?? '',
+                        'created_at' => $parts[1] ?? '',
+                        'image_ref' => $imageRef,
+                    ];
+                })
+                ->filter(fn ($image) => ! empty($image['tag']));
+
+            // Separate images into categories
+            // PR images (pr-*) are always deleted
+            // Build images (*-build) are cleaned up to match retained regular images
+            $prImages = $images->filter(fn ($image) => str_starts_with($image['tag'], 'pr-'));
+            $buildImages = $images->filter(fn ($image) => ! str_starts_with($image['tag'], 'pr-') && str_ends_with($image['tag'], '-build'));
+            $regularImages = $images->filter(fn ($image) => ! str_starts_with($image['tag'], 'pr-') && ! str_ends_with($image['tag'], '-build'));
+
+            // Always delete all PR images
+            foreach ($prImages as $image) {
+                $deleteCommand = "docker rmi {$image['image_ref']} 2>/dev/null || true";
+                $deleteOutput = instant_remote_process([$deleteCommand], $server, false);
+                $cleanupLog[] = [
+                    'command' => $deleteCommand,
+                    'output' => $deleteOutput ?? 'PR image removed or was in use',
+                ];
+            }
+
+            // Filter out current running image from regular images and sort by creation date
+            $sortedRegularImages = $regularImages
+                ->filter(fn ($image) => $image['tag'] !== $currentTag)
+                ->sortByDesc('created_at')
+                ->values();
+
+            // Keep only N images (imagesToKeep), delete the rest
+            $imagesToDelete = $sortedRegularImages->skip($imagesToKeep);
+
+            foreach ($imagesToDelete as $image) {
+                $deleteCommand = "docker rmi {$image['image_ref']} 2>/dev/null || true";
+                $deleteOutput = instant_remote_process([$deleteCommand], $server, false);
+                $cleanupLog[] = [
+                    'command' => $deleteCommand,
+                    'output' => $deleteOutput ?? 'Image removed or was in use',
+                ];
+            }
+
+            // Clean up build images (-build suffix) that don't correspond to retained regular images
+            // Build images are intermediate artifacts (e.g. Nixpacks) not used by running containers.
+            // If a build is in progress, docker rmi will fail silently since the image is in use.
+            $keptTags = $sortedRegularImages->take($imagesToKeep)->pluck('tag');
+            if (! empty($currentTag)) {
+                $keptTags = $keptTags->push($currentTag);
+            }
+
+            foreach ($buildImages as $image) {
+                $baseTag = preg_replace('/-build$/', '', $image['tag']);
+                if (! $keptTags->contains($baseTag)) {
+                    $deleteCommand = "docker rmi {$image['image_ref']} 2>/dev/null || true";
+                    $deleteOutput = instant_remote_process([$deleteCommand], $server, false);
+                    $cleanupLog[] = [
+                        'command' => $deleteCommand,
+                        'output' => $deleteOutput ?? 'Build image removed or was in use',
+                    ];
+                }
+            }
+        }
+
+        return $cleanupLog;
+    }
 }

app/Actions/Server/DeleteServer.php

@@ -2,16 +2,81 @@
 
 namespace App\Actions\Server;
 
+use App\Models\CloudProviderToken;
 use App\Models\Server;
+use App\Models\Team;
+use App\Notifications\Server\HetznerDeletionFailed;
+use App\Services\HetznerService;
 use Lorisleiva\Actions\Concerns\AsAction;
 
 class DeleteServer
 {
     use AsAction;
 
-    public function handle(Server $server)
+    public function handle(int $serverId, bool $deleteFromHetzner = false, ?int $hetznerServerId = null, ?int $cloudProviderTokenId = null, ?int $teamId = null)
     {
-        StopSentinel::run($server);
-        $server->forceDelete();
+        $server = Server::withTrashed()->find($serverId);
+
+        // Delete from Hetzner even if server is already gone from Coolify
+        if ($deleteFromHetzner && ($hetznerServerId || ($server && $server->hetzner_server_id))) {
+            $this->deleteFromHetznerById(
+                $hetznerServerId ?? $server->hetzner_server_id,
+                $cloudProviderTokenId ?? $server->cloud_provider_token_id,
+                $teamId ?? $server->team_id
+            );
+        }
+
+        // If server is already deleted from Coolify, skip this part
+        if (! $server) {
+            return; // Server already force deleted from Coolify
+        }
+
+        try {
+            $server->forceDelete();
+        } catch (\Throwable $e) {
+            logger()->error('Failed to force delete server from Coolify', [
+                'error' => $e->getMessage(),
+                'server_id' => $server->id,
+            ]);
+        }
+    }
+
+    private function deleteFromHetznerById(int $hetznerServerId, ?int $cloudProviderTokenId, int $teamId): void
+    {
+        try {
+            // Use the provided token, or fallback to first available team token
+            $token = null;
+
+            if ($cloudProviderTokenId) {
+                $token = CloudProviderToken::find($cloudProviderTokenId);
+            }
+
+            if (! $token) {
+                $token = CloudProviderToken::where('team_id', $teamId)
+                    ->where('provider', 'hetzner')
+                    ->first();
+            }
+
+            if (! $token) {
+
+                return;
+            }
+
+            $hetznerService = new HetznerService($token->token);
+            $hetznerService->deleteServer($hetznerServerId);
+
+        } catch (\Throwable $e) {
+
+            // Log the error but don't prevent the server from being deleted from Coolify
+            logger()->error('Failed to delete server from Hetzner', [
+                'error' => $e->getMessage(),
+                'hetzner_server_id' => $hetznerServerId,
+                'team_id' => $teamId,
+            ]);
+
+            // Notify the team about the failure
+            $team = Team::find($teamId);
+            $team?->notify(new HetznerDeletionFailed($hetznerServerId, $teamId, $e->getMessage()));
+        }
     }
 }

app/Actions/Server/InstallDocker.php

@@ -4,7 +4,6 @@ namespace App\Actions\Server;
 
 use App\Helpers\SslHelper;
 use App\Models\Server;
-use App\Models\SslCertificate;
 use App\Models\StandaloneDocker;
 use Lorisleiva\Actions\Concerns\AsAction;
 
@@ -14,13 +13,12 @@ class InstallDocker
 
     public function handle(Server $server)
     {
-        $dockerVersion = config('constants.docker.minimum_required_version');
         $supported_os_type = $server->validateOS();
         if (! $supported_os_type) {
             throw new \Exception('Server OS type is not supported for automated installation. Please install Docker manually before continuing: <a target="_blank" class="underline" href="https://coolify.io/docs/installation#manually">documentation</a>.');
         }
 
-        if (! SslCertificate::where('server_id', $server->id)->where('is_ca_certificate', true)->exists()) {
+        if (! $server->sslCertificates()->where('is_ca_certificate', true)->exists()) {
             $serverCert = SslHelper::generateSslCertificate(
                 commonName: 'Coolify CA Certificate',
                 serverId: $server->id,
@@ -29,12 +27,14 @@ class InstallDocker
             );
             $caCertPath = config('constants.coolify.base_config_path').'/ssl/';
 
+            $base64Cert = base64_encode($serverCert->ssl_certificate);
+
             $commands = collect([
                 "mkdir -p $caCertPath",
                 "chown -R 9999:root $caCertPath",
                 "chmod -R 700 $caCertPath",
                 "rm -rf $caCertPath/coolify-ca.crt",
-                "echo '{$serverCert->ssl_certificate}' > $caCertPath/coolify-ca.crt",
+                "echo '{$base64Cert}' | base64 -d | tee $caCertPath/coolify-ca.crt > /dev/null",
                 "chmod 644 $caCertPath/coolify-ca.crt",
             ]);
             remote_process($commands, $server);
@@ -58,8 +58,6 @@ class InstallDocker
         $command = collect([]);
         if (isDev() && $server->id === 0) {
             $command = $command->merge([
-                "echo 'Installing Prerequisites...'",
-                'sleep 1',
                 "echo 'Installing Docker Engine...'",
                 "echo 'Configuring Docker Engine (merging existing configuration with the required)...'",
                 'sleep 4',
@@ -69,38 +67,23 @@ class InstallDocker
 
             return remote_process($command, $server);
         } else {
-            if ($supported_os_type->contains('debian')) {
-                $command = $command->merge([
-                    "echo 'Installing Prerequisites...'",
-                    'apt-get update -y',
-                    'command -v curl >/dev/null || apt install -y curl',
-                    'command -v wget >/dev/null || apt install -y wget',
-                    'command -v git >/dev/null || apt install -y git',
-                    'command -v jq >/dev/null || apt install -y jq',
-                ]);
-            } elseif ($supported_os_type->contains('rhel')) {
-                $command = $command->merge([
-                    "echo 'Installing Prerequisites...'",
-                    'command -v curl >/dev/null || dnf install -y curl',
-                    'command -v wget >/dev/null || dnf install -y wget',
-                    'command -v git >/dev/null || dnf install -y git',
-                    'command -v jq >/dev/null || dnf install -y jq',
-                ]);
-            } elseif ($supported_os_type->contains('sles')) {
-                $command = $command->merge([
-                    "echo 'Installing Prerequisites...'",
-                    'zypper update -y',
-                    'command -v curl >/dev/null || zypper install -y curl',
-                    'command -v wget >/dev/null || zypper install -y wget',
-                    'command -v git >/dev/null || zypper install -y git',
-                    'command -v jq >/dev/null || zypper install -y jq',
-                ]);
-            } else {
-                throw new \Exception('Unsupported OS');
-            }
             $command = $command->merge([
                 "echo 'Installing Docker Engine...'",
-                "curl https://releases.rancher.com/install-docker/{$dockerVersion}.sh | sh || curl https://get.docker.com | sh -s -- --version {$dockerVersion}",
+            ]);
+
+            if ($supported_os_type->contains('debian')) {
+                $command = $command->merge([$this->getDebianDockerInstallCommand()]);
+            } elseif ($supported_os_type->contains('rhel')) {
+                $command = $command->merge([$this->getRhelDockerInstallCommand()]);
+            } elseif ($supported_os_type->contains('sles')) {
+                $command = $command->merge([$this->getSuseDockerInstallCommand()]);
+            } elseif ($supported_os_type->contains('arch')) {
+                $command = $command->merge([$this->getArchDockerInstallCommand()]);
+            } else {
+                $command = $command->merge([$this->getGenericDockerInstallCommand()]);
+            }
+
+            $command = $command->merge([
                 "echo 'Configuring Docker Engine (merging existing configuration with the required)...'",
                 'test -s /etc/docker/daemon.json && cp /etc/docker/daemon.json "/etc/docker/daemon.json.original-$(date +"%Y%m%d-%H%M%S")"',
                 "test ! -s /etc/docker/daemon.json && echo '{$config}' | base64 -d | tee /etc/docker/daemon.json > /dev/null",
@@ -129,4 +112,50 @@ class InstallDocker
             return remote_process($command, $server);
         }
     }
+
+    private function getDebianDockerInstallCommand(): string
+    {
+        return 'curl -fsSL https://get.docker.com | sh || ('.
+            '. /etc/os-release && '.
+            'install -m 0755 -d /etc/apt/keyrings && '.
+            'curl -fsSL https://download.docker.com/linux/${ID}/gpg -o /etc/apt/keyrings/docker.asc && '.
+            'chmod a+r /etc/apt/keyrings/docker.asc && '.
+            'echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/${ID} ${VERSION_CODENAME} stable" > /etc/apt/sources.list.d/docker.list && '.
+            'apt-get update && '.
+            'apt-get install -y docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin'.
+            ')';
+    }
+
+    private function getRhelDockerInstallCommand(): string
+    {
+        return 'curl -fsSL https://get.docker.com | sh || ('.
+            'dnf config-manager --add-repo https://download.docker.com/linux/centos/docker-ce.repo && '.
+            'dnf install -y docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin && '.
+            'systemctl start docker && '.
+            'systemctl enable docker'.
+            ')';
+    }
+
+    private function getSuseDockerInstallCommand(): string
+    {
+        return 'curl -fsSL https://get.docker.com | sh || ('.
+            'zypper addrepo https://download.docker.com/linux/sles/docker-ce.repo && '.
+            'zypper refresh && '.
+            'zypper install -y --no-confirm docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin && '.
+            'systemctl start docker && '.
+            'systemctl enable docker'.
+            ')';
+    }
+
+    private function getArchDockerInstallCommand(): string
+    {
+        return 'pacman -Syu --noconfirm --needed docker docker-compose && '.
+            'systemctl enable docker.service && '.
+            'systemctl start docker.service';
+    }
+
+    private function getGenericDockerInstallCommand(): string
+    {
+        return 'curl -fsSL https://get.docker.com | sh';
+    }
 }

app/Actions/Server/InstallPrerequisites.php

@@ -0,0 +1,64 @@
+<?php
+
+namespace App\Actions\Server;
+
+use App\Models\Server;
+use Lorisleiva\Actions\Concerns\AsAction;
+
+class InstallPrerequisites
+{
+    use AsAction;
+
+    public string $jobQueue = 'high';
+
+    public function handle(Server $server)
+    {
+        $supported_os_type = $server->validateOS();
+        if (! $supported_os_type) {
+            throw new \Exception('Server OS type is not supported for automated installation. Please install prerequisites manually.');
+        }
+
+        $command = collect([]);
+
+        if ($supported_os_type->contains('debian')) {
+            $command = $command->merge([
+                "echo 'Installing Prerequisites...'",
+                'apt-get update -y',
+                'command -v curl >/dev/null || apt install -y curl',
+                'command -v wget >/dev/null || apt install -y wget',
+                'command -v git >/dev/null || apt install -y git',
+                'command -v jq >/dev/null || apt install -y jq',
+            ]);
+        } elseif ($supported_os_type->contains('rhel')) {
+            $command = $command->merge([
+                "echo 'Installing Prerequisites...'",
+                'command -v curl >/dev/null || dnf install -y curl',
+                'command -v wget >/dev/null || dnf install -y wget',
+                'command -v git >/dev/null || dnf install -y git',
+                'command -v jq >/dev/null || dnf install -y jq',
+            ]);
+        } elseif ($supported_os_type->contains('sles')) {
+            $command = $command->merge([
+                "echo 'Installing Prerequisites...'",
+                'zypper update -y',
+                'command -v curl >/dev/null || zypper install -y curl',
+                'command -v wget >/dev/null || zypper install -y wget',
+                'command -v git >/dev/null || zypper install -y git',
+                'command -v jq >/dev/null || zypper install -y jq',
+            ]);
+        } elseif ($supported_os_type->contains('arch')) {
+            // Use -Syu for full system upgrade to avoid partial upgrade issues on Arch Linux
+            // --needed flag skips packages that are already installed and up-to-date
+            $command = $command->merge([
+                "echo 'Installing Prerequisites for Arch Linux...'",
+                'pacman -Syu --noconfirm --needed curl wget git jq',
+            ]);
+        } else {
+            throw new \Exception('Unsupported OS type for prerequisites installation');
+        }
+
+        $command->push("echo 'Prerequisites installed successfully.'");
+
+        return remote_process($command, $server);
+    }
+}

app/Actions/Server/ResourcesCheck.php

@@ -1,41 +0,0 @@
-<?php
-
-namespace App\Actions\Server;
-
-use App\Models\Application;
-use App\Models\ServiceApplication;
-use App\Models\ServiceDatabase;
-use App\Models\StandaloneClickhouse;
-use App\Models\StandaloneDragonfly;
-use App\Models\StandaloneKeydb;
-use App\Models\StandaloneMariadb;
-use App\Models\StandaloneMongodb;
-use App\Models\StandaloneMysql;
-use App\Models\StandalonePostgresql;
-use App\Models\StandaloneRedis;
-use Lorisleiva\Actions\Concerns\AsAction;
-
-class ResourcesCheck
-{
-    use AsAction;
-
-    public function handle()
-    {
-        $seconds = 60;
-        try {
-            Application::where('last_online_at', '<', now()->subSeconds($seconds))->update(['status' => 'exited']);
-            ServiceApplication::where('last_online_at', '<', now()->subSeconds($seconds))->update(['status' => 'exited']);
-            ServiceDatabase::where('last_online_at', '<', now()->subSeconds($seconds))->update(['status' => 'exited']);
-            StandalonePostgresql::where('last_online_at', '<', now()->subSeconds($seconds))->update(['status' => 'exited']);
-            StandaloneRedis::where('last_online_at', '<', now()->subSeconds($seconds))->update(['status' => 'exited']);
-            StandaloneMongodb::where('last_online_at', '<', now()->subSeconds($seconds))->update(['status' => 'exited']);
-            StandaloneMysql::where('last_online_at', '<', now()->subSeconds($seconds))->update(['status' => 'exited']);
-            StandaloneMariadb::where('last_online_at', '<', now()->subSeconds($seconds))->update(['status' => 'exited']);
-            StandaloneKeydb::where('last_online_at', '<', now()->subSeconds($seconds))->update(['status' => 'exited']);
-            StandaloneDragonfly::where('last_online_at', '<', now()->subSeconds($seconds))->update(['status' => 'exited']);
-            StandaloneClickhouse::where('last_online_at', '<', now()->subSeconds($seconds))->update(['status' => 'exited']);
-        } catch (\Throwable $e) {
-            return handleError($e);
-        }
-    }
-}

app/Actions/Server/ServerCheck.php

@@ -1,268 +0,0 @@
-<?php
-
-namespace App\Actions\Server;
-
-use App\Actions\Database\StartDatabaseProxy;
-use App\Actions\Proxy\CheckProxy;
-use App\Actions\Proxy\StartProxy;
-use App\Jobs\CheckAndStartSentinelJob;
-use App\Jobs\ServerStorageCheckJob;
-use App\Models\Application;
-use App\Models\ApplicationPreview;
-use App\Models\Server;
-use App\Models\Service;
-use App\Models\ServiceApplication;
-use App\Models\ServiceDatabase;
-use App\Notifications\Container\ContainerRestarted;
-use Illuminate\Support\Arr;
-use Lorisleiva\Actions\Concerns\AsAction;
-
-class ServerCheck
-{
-    use AsAction;
-
-    public Server $server;
-
-    public bool $isSentinel = false;
-
-    public $containers;
-
-    public $databases;
-
-    public function handle(Server $server, $data = null)
-    {
-        $this->server = $server;
-        try {
-            if ($this->server->isFunctional() === false) {
-                return 'Server is not functional.';
-            }
-
-            if (! $this->server->isSwarmWorker() && ! $this->server->isBuildServer()) {
-
-                if (isset($data)) {
-                    $data = collect($data);
-
-                    $this->server->sentinelHeartbeat();
-
-                    $this->containers = collect(data_get($data, 'containers'));
-
-                    $filesystemUsageRoot = data_get($data, 'filesystem_usage_root.used_percentage');
-                    ServerStorageCheckJob::dispatch($this->server, $filesystemUsageRoot);
-
-                    $containerReplicates = null;
-                    $this->isSentinel = true;
-                } else {
-                    ['containers' => $this->containers, 'containerReplicates' => $containerReplicates] = $this->server->getContainers();
-                    // ServerStorageCheckJob::dispatch($this->server);
-                }
-
-                if (is_null($this->containers)) {
-                    return 'No containers found.';
-                }
-
-                if (isset($containerReplicates)) {
-                    foreach ($containerReplicates as $containerReplica) {
-                        $name = data_get($containerReplica, 'Name');
-                        $this->containers = $this->containers->map(function ($container) use ($name, $containerReplica) {
-                            if (data_get($container, 'Spec.Name') === $name) {
-                                $replicas = data_get($containerReplica, 'Replicas');
-                                $running = str($replicas)->explode('/')[0];
-                                $total = str($replicas)->explode('/')[1];
-                                if ($running === $total) {
-                                    data_set($container, 'State.Status', 'running');
-                                    data_set($container, 'State.Health.Status', 'healthy');
-                                } else {
-                                    data_set($container, 'State.Status', 'starting');
-                                    data_set($container, 'State.Health.Status', 'unhealthy');
-                                }
-                            }
-
-                            return $container;
-                        });
-                    }
-                }
-                $this->checkContainers();
-
-                if ($this->server->isSentinelEnabled() && $this->isSentinel === false) {
-                    CheckAndStartSentinelJob::dispatch($this->server);
-                }
-
-                if ($this->server->isLogDrainEnabled()) {
-                    $this->checkLogDrainContainer();
-                }
-
-                if ($this->server->proxySet() && ! $this->server->proxy->force_stop) {
-                    $foundProxyContainer = $this->containers->filter(function ($value, $key) {
-                        if ($this->server->isSwarm()) {
-                            return data_get($value, 'Spec.Name') === 'coolify-proxy_traefik';
-                        } else {
-                            return data_get($value, 'Name') === '/coolify-proxy';
-                        }
-                    })->first();
-                    $proxyStatus = data_get($foundProxyContainer, 'State.Status', 'exited');
-                    if (! $foundProxyContainer || $proxyStatus !== 'running') {
-                        try {
-                            $shouldStart = CheckProxy::run($this->server);
-                            if ($shouldStart) {
-                                StartProxy::run($this->server, async: false);
-                                $this->server->team?->notify(new ContainerRestarted('coolify-proxy', $this->server));
-                            }
-                        } catch (\Throwable $e) {
-                        }
-                    } else {
-                        $this->server->proxy->status = data_get($foundProxyContainer, 'State.Status');
-                        $this->server->save();
-                        $connectProxyToDockerNetworks = connectProxyToNetworks($this->server);
-                        instant_remote_process($connectProxyToDockerNetworks, $this->server, false);
-                    }
-                }
-            }
-        } catch (\Throwable $e) {
-            return handleError($e);
-        }
-    }
-
-    private function checkLogDrainContainer()
-    {
-        $foundLogDrainContainer = $this->containers->filter(function ($value, $key) {
-            return data_get($value, 'Name') === '/coolify-log-drain';
-        })->first();
-        if ($foundLogDrainContainer) {
-            $status = data_get($foundLogDrainContainer, 'State.Status');
-            if ($status !== 'running') {
-                StartLogDrain::dispatch($this->server);
-            }
-        } else {
-            StartLogDrain::dispatch($this->server);
-        }
-    }
-
-    private function checkContainers()
-    {
-        foreach ($this->containers as $container) {
-            if ($this->isSentinel) {
-                $labels = Arr::undot(data_get($container, 'labels'));
-            } else {
-                if ($this->server->isSwarm()) {
-                    $labels = Arr::undot(data_get($container, 'Spec.Labels'));
-                } else {
-                    $labels = Arr::undot(data_get($container, 'Config.Labels'));
-                }
-            }
-            $managed = data_get($labels, 'coolify.managed');
-            if (! $managed) {
-                continue;
-            }
-            $uuid = data_get($labels, 'coolify.name');
-            if (! $uuid) {
-                $uuid = data_get($labels, 'com.docker.compose.service');
-            }
-
-            if ($this->isSentinel) {
-                $containerStatus = data_get($container, 'state');
-                $containerHealth = data_get($container, 'health_status');
-            } else {
-                $containerStatus = data_get($container, 'State.Status');
-                $containerHealth = data_get($container, 'State.Health.Status', 'unhealthy');
-            }
-            $containerStatus = "$containerStatus ($containerHealth)";
-
-            $applicationId = data_get($labels, 'coolify.applicationId');
-            $serviceId = data_get($labels, 'coolify.serviceId');
-            $databaseId = data_get($labels, 'coolify.databaseId');
-            $pullRequestId = data_get($labels, 'coolify.pullRequestId');
-
-            if ($applicationId) {
-                // Application
-                if ($pullRequestId != 0) {
-                    if (str($applicationId)->contains('-')) {
-                        $applicationId = str($applicationId)->before('-');
-                    }
-                    $preview = ApplicationPreview::where('application_id', $applicationId)->where('pull_request_id', $pullRequestId)->first();
-                    if ($preview) {
-                        $preview->update(['status' => $containerStatus]);
-                    }
-                } else {
-                    $application = Application::where('id', $applicationId)->first();
-                    if ($application) {
-                        $application->update([
-                            'status' => $containerStatus,
-                            'last_online_at' => now(),
-                        ]);
-                    }
-                }
-            } elseif (isset($serviceId)) {
-                // Service
-                $subType = data_get($labels, 'coolify.service.subType');
-                $subId = data_get($labels, 'coolify.service.subId');
-                $service = Service::where('id', $serviceId)->first();
-                if (! $service) {
-                    continue;
-                }
-                if ($subType === 'application') {
-                    $service = ServiceApplication::where('id', $subId)->first();
-                } else {
-                    $service = ServiceDatabase::where('id', $subId)->first();
-                }
-                if ($service) {
-                    $service->update([
-                        'status' => $containerStatus,
-                        'last_online_at' => now(),
-                    ]);
-                    if ($subType === 'database') {
-                        $isPublic = data_get($service, 'is_public');
-                        if ($isPublic) {
-                            $foundTcpProxy = $this->containers->filter(function ($value, $key) use ($uuid) {
-                                if ($this->isSentinel) {
-                                    return data_get($value, 'name') === $uuid.'-proxy';
-                                } else {
-
-                                    if ($this->server->isSwarm()) {
-                                        return data_get($value, 'Spec.Name') === "coolify-proxy_$uuid";
-                                    } else {
-                                        return data_get($value, 'Name') === "/$uuid-proxy";
-                                    }
-                                }
-                            })->first();
-                            if (! $foundTcpProxy) {
-                                StartDatabaseProxy::run($service);
-                            }
-                        }
-                    }
-                }
-            } else {
-                // Database
-                if (is_null($this->databases)) {
-                    $this->databases = $this->server->databases();
-                }
-                $database = $this->databases->where('uuid', $uuid)->first();
-                if ($database) {
-                    $database->update([
-                        'status' => $containerStatus,
-                        'last_online_at' => now(),
-                    ]);
-
-                    $isPublic = data_get($database, 'is_public');
-                    if ($isPublic) {
-                        $foundTcpProxy = $this->containers->filter(function ($value, $key) use ($uuid) {
-                            if ($this->isSentinel) {
-                                return data_get($value, 'name') === $uuid.'-proxy';
-                            } else {
-                                if ($this->server->isSwarm()) {
-                                    return data_get($value, 'Spec.Name') === "coolify-proxy_$uuid";
-                                } else {
-
-                                    return data_get($value, 'Name') === "/$uuid-proxy";
-                                }
-                            }
-                        })->first();
-                        if (! $foundTcpProxy) {
-                            StartDatabaseProxy::run($database);
-                            // $this->server->team?->notify(new ContainerRestarted("TCP Proxy for database", $this->server));
-                        }
-                    }
-                }
-            }
-        }
-    }
-}

app/Actions/Server/StartLogDrain.php

@@ -3,6 +3,7 @@
 namespace App\Actions\Server;
 
 use App\Models\Server;
+use App\Models\Service;
 use Lorisleiva\Actions\Concerns\AsAction;
 
 class StartLogDrain
@@ -177,6 +178,19 @@ Files:
             $parsers_config = $config_path.'/parsers.conf';
             $compose_path = $config_path.'/docker-compose.yml';
             $readme_path = $config_path.'/README.md';
+            if ($type === 'newrelic') {
+                $envContent = "LICENSE_KEY={$license_key}\nBASE_URI={$base_uri}\n";
+            } elseif ($type === 'highlight') {
+                $envContent = "HIGHLIGHT_PROJECT_ID={$server->settings->logdrain_highlight_project_id}\n";
+            } elseif ($type === 'axiom') {
+                $envContent = "AXIOM_DATASET_NAME={$server->settings->logdrain_axiom_dataset_name}\nAXIOM_API_KEY={$server->settings->logdrain_axiom_api_key}\n";
+            } elseif ($type === 'custom') {
+                $envContent = '';
+            } else {
+                throw new \Exception('Unknown log drain type.');
+            }
+            $envEncoded = base64_encode($envContent);
+
             $command = [
                 "echo 'Saving configuration'",
                 "mkdir -p $config_path",
@@ -184,38 +198,33 @@ Files:
                 "echo '{$config}' | base64 -d | tee $fluent_bit_config > /dev/null",
                 "echo '{$compose}' | base64 -d | tee $compose_path > /dev/null",
                 "echo '{$readme}' | base64 -d | tee $readme_path > /dev/null",
-                "test -f $config_path/.env && rm $config_path/.env",
-            ];
-            if ($type === 'newrelic') {
-                $add_envs_command = [
-                    "echo LICENSE_KEY=$license_key >> $config_path/.env",
-                    "echo BASE_URI=$base_uri >> $config_path/.env",
-                ];
-            } elseif ($type === 'highlight') {
-                $add_envs_command = [
-                    "echo HIGHLIGHT_PROJECT_ID={$server->settings->logdrain_highlight_project_id} >> $config_path/.env",
-                ];
-            } elseif ($type === 'axiom') {
-                $add_envs_command = [
-                    "echo AXIOM_DATASET_NAME={$server->settings->logdrain_axiom_dataset_name} >> $config_path/.env",
-                    "echo AXIOM_API_KEY={$server->settings->logdrain_axiom_api_key} >> $config_path/.env",
-                ];
-            } elseif ($type === 'custom') {
-                $add_envs_command = [
-                    "touch $config_path/.env",
-                ];
-            } else {
-                throw new \Exception('Unknown log drain type.');
-            }
-            $restart_command = [
+                "echo '{$envEncoded}' | base64 -d | tee $config_path/.env > /dev/null",
                 "echo 'Starting Fluent Bit'",
                 "cd $config_path && docker compose up -d",
             ];
-            $command = array_merge($command, $add_envs_command, $restart_command);
+            $command = array_merge($command, $this->logDrainNetworkConnectCommands($server));
 
             return instant_remote_process($command, $server);
         } catch (\Throwable $e) {
             return handleError($e);
         }
     }
+
+    private function logDrainNetworkConnectCommands(Server $server): array
+    {
+        if (! $server->isLogDrainEnabled()) {
+            return [];
+        }
+
+        return $server->services()
+            ->with('destination')
+            ->where('connect_to_docker_network', true)
+            ->get()
+            ->map(fn (Service $service) => data_get($service, 'destination.network'))
+            ->filter()
+            ->unique()
+            ->map(fn (string $network) => 'docker network connect '.escapeshellarg($network).' coolify-log-drain >/dev/null 2>&1 || true')
+            ->values()
+            ->all();
+    }
 }

app/Actions/Server/StartSentinel.php

@@ -2,6 +2,7 @@
 
 namespace App\Actions\Server;
 
+use App\Events\SentinelRestarted;
 use App\Models\Server;
 use Lorisleiva\Actions\Concerns\AsAction;
 
@@ -9,7 +10,7 @@ class StartSentinel
 {
     use AsAction;
 
-    public function handle(Server $server, bool $restart = false, ?string $latestVersion = null)
+    public function handle(Server $server, bool $restart = false, ?string $latestVersion = null, ?string $customImage = null)
     {
         if ($server->isSwarm() || $server->isBuildServer()) {
             return;
@@ -21,11 +22,11 @@ class StartSentinel
         $metricsHistory = data_get($server, 'settings.sentinel_metrics_history_days');
         $refreshRate = data_get($server, 'settings.sentinel_metrics_refresh_rate_seconds');
         $pushInterval = data_get($server, 'settings.sentinel_push_interval_seconds');
-        $token = data_get($server, 'settings.sentinel_token');
+        $token = $server->settings->ensureValidSentinelToken();
         $endpoint = data_get($server, 'settings.sentinel_custom_url');
         $debug = data_get($server, 'settings.is_sentinel_debug_enabled');
         $mountDir = '/data/coolify/sentinel';
-        $image = config('constants.coolify.registry_url').'/coollabsio/sentinel:'.$version;
+        $image = coolifyRegistryUrl().'/coollabsio/sentinel:'.$version;
         if (! $endpoint) {
             throw new \RuntimeException('You should set FQDN in Instance Settings.');
         }
@@ -43,10 +44,12 @@ class StartSentinel
         ];
         if (isDev()) {
             // data_set($environments, 'DEBUG', 'true');
-            // $image = 'sentinel';
+            if ($customImage && ! empty($customImage)) {
+                $image = $customImage;
+            }
             $mountDir = '/var/lib/docker/volumes/coolify_dev_coolify_data/_data/sentinel';
         }
-        $dockerEnvironments = '-e "'.implode('" -e "', array_map(fn ($key, $value) => "$key=$value", array_keys($environments), $environments)).'"';
+        $dockerEnvironments = implode(' ', array_map(fn ($key, $value) => '-e '.escapeshellarg("$key=$value"), array_keys($environments), $environments));
         $dockerLabels = implode(' ', array_map(fn ($key, $value) => "$key=$value", array_keys($labels), $labels));
         $dockerCommand = "docker run -d $dockerEnvironments --name coolify-sentinel -v /var/run/docker.sock:/var/run/docker.sock -v $mountDir:/app/db --pid host --health-cmd \"curl --fail http://127.0.0.1:8888/api/health || exit 1\" --health-interval 10s --health-retries 3 --add-host=host.docker.internal:host-gateway --label $dockerLabels $image";
 
@@ -61,5 +64,8 @@ class StartSentinel
         $server->settings->is_sentinel_enabled = true;
         $server->settings->save();
         $server->sentinelHeartbeat();
+
+        // Dispatch event to notify UI components
+        SentinelRestarted::dispatch($server, $version);
     }
 }

app/Actions/Server/UpdateCoolify.php

@@ -2,8 +2,9 @@
 
 namespace App\Actions\Server;
 
-use App\Jobs\PullHelperImageJob;
 use App\Models\Server;
+use Illuminate\Support\Facades\Http;
+use Illuminate\Support\Facades\Log;
 use Illuminate\Support\Sleep;
 use Lorisleiva\Actions\Concerns\AsAction;
 
@@ -29,8 +30,59 @@ class UpdateCoolify
         if (! $this->server) {
             return;
         }
-        CleanupDocker::dispatch($this->server, false, false);
-        $this->latestVersion = get_latest_version_of_coolify();
+
+        // Fetch fresh version from CDN instead of using cache
+        try {
+            $response = Http::retry(3, 1000)->timeout(10)
+                ->get(config('constants.coolify.versions_url'));
+
+            if ($response->successful()) {
+                $versions = $response->json();
+                $this->latestVersion = data_get($versions, 'coolify.v4.version');
+            } else {
+                // Fallback to cache if CDN unavailable
+                $cacheVersion = get_latest_version_of_coolify();
+
+                // Validate cache version against current running version
+                if ($cacheVersion && version_compare($cacheVersion, config('constants.coolify.version'), '<')) {
+                    Log::error('Failed to fetch fresh version from CDN and cache is corrupted/outdated', [
+                        'cached_version' => $cacheVersion,
+                        'current_version' => config('constants.coolify.version'),
+                    ]);
+                    throw new \Exception(
+                        'Cannot determine latest version: CDN unavailable and cache version '.
+                        "({$cacheVersion}) is older than running version (".config('constants.coolify.version').')'
+                    );
+                }
+
+                $this->latestVersion = $cacheVersion;
+                Log::warning('Failed to fetch fresh version from CDN (unsuccessful response), using validated cache', [
+                    'version' => $cacheVersion,
+                ]);
+            }
+        } catch (\Throwable $e) {
+            $cacheVersion = get_latest_version_of_coolify();
+
+            // Validate cache version against current running version
+            if ($cacheVersion && version_compare($cacheVersion, config('constants.coolify.version'), '<')) {
+                Log::error('Failed to fetch fresh version from CDN and cache is corrupted/outdated', [
+                    'error' => $e->getMessage(),
+                    'cached_version' => $cacheVersion,
+                    'current_version' => config('constants.coolify.version'),
+                ]);
+                throw new \Exception(
+                    'Cannot determine latest version: CDN unavailable and cache version '.
+                    "({$cacheVersion}) is older than running version (".config('constants.coolify.version').')'
+                );
+            }
+
+            $this->latestVersion = $cacheVersion;
+            Log::warning('Failed to fetch fresh version from CDN, using validated cache', [
+                'error' => $e->getMessage(),
+                'version' => $cacheVersion,
+            ]);
+        }
+
         $this->currentVersion = config('constants.coolify.version');
         if (! $manual_update) {
             if (! $settings->is_auto_update_enabled) {
@@ -43,6 +95,20 @@ class UpdateCoolify
                 return;
             }
         }
+
+        // ALWAYS check for downgrades (even for manual updates)
+        if (version_compare($this->latestVersion, $this->currentVersion, '<')) {
+            Log::error('Downgrade prevented', [
+                'target_version' => $this->latestVersion,
+                'current_version' => $this->currentVersion,
+                'manual_update' => $manual_update,
+            ]);
+            throw new \Exception(
+                "Cannot downgrade from {$this->currentVersion} to {$this->latestVersion}. ".
+                'If you need to downgrade, please do so manually via Docker commands.'
+            );
+        }
+
         $this->update();
         $settings->new_version_available = false;
         $settings->save();
@@ -50,14 +116,16 @@ class UpdateCoolify
 
     private function update()
     {
-        PullHelperImageJob::dispatch($this->server);
-
-        $image = config('constants.coolify.registry_url').'/coollabsio/coolify:'.$this->latestVersion;
-        instant_remote_process(["docker pull -q $image"], $this->server, false);
+        $latestHelperImageVersion = getHelperVersion();
+        $upgradeScriptUrl = config('constants.coolify.upgrade_script_url');
+        $registryUrl = coolifyRegistryUrl();
 
         remote_process([
-            'curl -fsSL https://cdn.coollabs.io/coolify/upgrade.sh -o /data/coolify/source/upgrade.sh',
-            "bash /data/coolify/source/upgrade.sh $this->latestVersion",
+            "curl -fsSL {$upgradeScriptUrl} -o /data/coolify/source/upgrade.sh",
+            'bash /data/coolify/source/upgrade.sh '.
+                escapeshellarg($this->latestVersion).' '.
+                escapeshellarg($latestHelperImageVersion).' '.
+                escapeshellarg($registryUrl),
         ], $this->server);
     }
 }

app/Actions/Server/UpdatePackage.php

@@ -20,18 +20,43 @@ class UpdatePackage
                     'error' => 'Server is not reachable or not ready.',
                 ];
             }
+
+            // Validate that package name is provided when not updating all packages
+            if (! $all && ($package === null || $package === '')) {
+                return [
+                    'error' => "Package name required when 'all' is false.",
+                ];
+            }
+
+            // Sanitize package name to prevent command injection
+            // Only allow alphanumeric characters, hyphens, underscores, periods, plus signs, and colons
+            // These are valid characters in package names across most package managers
+            $sanitizedPackage = '';
+            if ($package !== null && ! $all) {
+                if (! preg_match('/^[a-zA-Z0-9._+:-]+$/', $package)) {
+                    return [
+                        'error' => 'Invalid package name. Package names can only contain alphanumeric characters, hyphens, underscores, periods, plus signs, and colons.',
+                    ];
+                }
+                $sanitizedPackage = escapeshellarg($package);
+            }
+
             switch ($packageManager) {
                 case 'zypper':
                     $commandAll = 'zypper update -y';
-                    $commandInstall = 'zypper install -y '.$package;
+                    $commandInstall = 'zypper install -y '.$sanitizedPackage;
                     break;
                 case 'dnf':
                     $commandAll = 'dnf update -y';
-                    $commandInstall = 'dnf update -y '.$package;
+                    $commandInstall = 'dnf update -y '.$sanitizedPackage;
                     break;
                 case 'apt':
                     $commandAll = 'apt update && apt upgrade -y';
-                    $commandInstall = 'apt install -y '.$package;
+                    $commandInstall = 'apt install -y '.$sanitizedPackage;
+                    break;
+                case 'pacman':
+                    $commandAll = 'pacman -Syu --noconfirm';
+                    $commandInstall = 'pacman -S --noconfirm '.$sanitizedPackage;
                     break;
                 default:
                     return [

app/Actions/Server/ValidatePrerequisites.php

@@ -0,0 +1,40 @@
+<?php
+
+namespace App\Actions\Server;
+
+use App\Models\Server;
+use Lorisleiva\Actions\Concerns\AsAction;
+
+class ValidatePrerequisites
+{
+    use AsAction;
+
+    public string $jobQueue = 'high';
+
+    /**
+     * Validate that required commands are available on the server.
+     *
+     * @return array{success: bool, missing: array<string>, found: array<string>}
+     */
+    public function handle(Server $server): array
+    {
+        $requiredCommands = ['git', 'curl', 'jq'];
+        $missing = [];
+        $found = [];
+
+        foreach ($requiredCommands as $cmd) {
+            $result = instant_remote_process(["command -v {$cmd}"], $server, false);
+            if (! $result) {
+                $missing[] = $cmd;
+            } else {
+                $found[] = $cmd;
+            }
+        }
+
+        return [
+            'success' => empty($missing),
+            'missing' => $missing,
+            'found' => $found,
+        ];
+    }
+}

app/Actions/Server/ValidateServer.php

@@ -30,7 +30,8 @@ class ValidateServer
         ]);
         ['uptime' => $this->uptime, 'error' => $error] = $server->validateConnection();
         if (! $this->uptime) {
-            $this->error = 'Server is not reachable. Please validate your configuration and connection.<br>Check this <a target="_blank" class="text-black underline dark:text-white" href="https://coolify.io/docs/knowledge-base/server/openssh">documentation</a> for further help. <br><br><div class="text-error">Error: '.$error.'</div>';
+            $sanitizedError = htmlspecialchars($error ?? '', ENT_QUOTES, 'UTF-8');
+            $this->error = 'Server is not reachable. Please validate your configuration and connection.<br>Check this <a target="_blank" class="text-black underline dark:text-white" href="https://coolify.io/docs/knowledge-base/server/openssh">documentation</a> for further help. <br><br><div class="text-error">Error: '.$sanitizedError.'</div>';
             $server->update([
                 'validation_logs' => $this->error,
             ]);
@@ -45,6 +46,16 @@ class ValidateServer
             throw new \Exception($this->error);
         }
 
+        $validationResult = $server->validatePrerequisites();
+        if (! $validationResult['success']) {
+            $missingCommands = implode(', ', $validationResult['missing']);
+            $this->error = "Prerequisites ({$missingCommands}) are not installed. Please install them before continuing or use the validation with installation endpoint.";
+            $server->update([
+                'validation_logs' => $this->error,
+            ]);
+            throw new \Exception($this->error);
+        }
+
         $this->docker_installed = $server->validateDockerEngine();
         $this->docker_compose_installed = $server->validateDockerCompose();
         if (! $this->docker_installed || ! $this->docker_compose_installed) {

app/Actions/Service/DeleteService.php

@@ -33,7 +33,7 @@ class DeleteService
                     }
                 }
                 foreach ($storagesToDelete as $storage) {
-                    $commands[] = "docker volume rm -f $storage->name";
+                    $commands[] = 'docker volume rm -f '.escapeshellarg($storage->name);
                 }
 
                 // Execute volume deletion first, this must be done first otherwise volumes will not be deleted.

app/Actions/Service/RestartService.php

@@ -13,8 +13,10 @@ class RestartService
 
     public function handle(Service $service, bool $pullLatestImages)
     {
-        StopService::run($service);
-
-        return StartService::run($service, $pullLatestImages);
+        return StartService::run(
+            service: $service,
+            pullLatestImages: $pullLatestImages,
+            stopBeforeStart: true,
+        );
     }
 }

app/Actions/Service/StartService.php

@@ -4,44 +4,80 @@ namespace App\Actions\Service;
 
 use App\Models\Service;
 use Lorisleiva\Actions\Concerns\AsAction;
+use Lorisleiva\Actions\Decorators\JobDecorator;
 use Symfony\Component\Yaml\Yaml;
 
 class StartService
 {
     use AsAction;
 
-    public string $jobQueue = 'high';
+    public function configureJob(JobDecorator $job): void
+    {
+        $job->onQueue(deployment_queue());
+    }
 
     public function handle(Service $service, bool $pullLatestImages = false, bool $stopBeforeStart = false)
     {
         $service->parse();
-        if ($stopBeforeStart) {
+        if ($this->shouldStopBeforeStarting($pullLatestImages, $stopBeforeStart)) {
             StopService::run(service: $service, dockerCleanup: false);
         }
         $service->saveComposeConfigs();
         $service->isConfigurationChanged(save: true);
-        $commands[] = 'cd '.$service->workdir();
-        $commands[] = "echo 'Saved configuration files to {$service->workdir()}.'";
+        $workdir = $service->workdir();
+        // $commands[] = "cd {$workdir}";
+        $commands[] = "echo 'Saved configuration files to {$workdir}.'";
+        // Ensure .env exists in the correct directory before docker compose tries to load it
+        // This is defensive programming - saveComposeConfigs() already creates it,
+        // but we guarantee it here in case of any edge cases or manual deployments
+        $commands[] = "touch {$workdir}/.env";
         if ($pullLatestImages) {
             $commands[] = "echo 'Pulling images.'";
-            $commands[] = 'docker compose pull';
+            $commands[] = "docker compose --project-directory {$workdir} pull";
         }
         if ($service->networks()->count() > 0) {
             $commands[] = "echo 'Creating Docker network.'";
             $commands[] = "docker network inspect $service->uuid >/dev/null 2>&1 || docker network create --attachable $service->uuid";
         }
         $commands[] = 'echo Starting service.';
-        $commands[] = 'docker compose up -d --remove-orphans --force-recreate --build';
+        $commands[] = "docker compose --project-directory {$workdir} -f {$workdir}/docker-compose.yml --project-name {$service->uuid} up -d --remove-orphans --force-recreate --build";
         $commands[] = "docker network connect $service->uuid coolify-proxy >/dev/null 2>&1 || true";
         if (data_get($service, 'connect_to_docker_network')) {
             $compose = data_get($service, 'docker_compose', []);
-            $network = $service->destination->network;
+            $safeNetwork = escapeshellarg($service->destination->network);
             $serviceNames = data_get(Yaml::parse($compose), 'services', []);
             foreach ($serviceNames as $serviceName => $serviceConfig) {
-                $commands[] = "docker network connect --alias {$serviceName}-{$service->uuid} $network {$serviceName}-{$service->uuid} >/dev/null 2>&1 || true";
+                $commands[] = "docker network connect --alias {$serviceName}-{$service->uuid} {$safeNetwork} {$serviceName}-{$service->uuid} >/dev/null 2>&1 || true";
             }
         }
+        $commands = array_merge($commands, $this->logDrainNetworkConnectCommands($service));
 
         return remote_process($commands, $service->server, type_uuid: $service->uuid, callEventOnFinish: 'ServiceStatusChanged');
     }
+
+    private function logDrainNetworkConnectCommands(Service $service): array
+    {
+        if (! data_get($service, 'connect_to_docker_network')) {
+            return [];
+        }
+
+        if (! $service->destination?->server?->isLogDrainEnabled()) {
+            return [];
+        }
+
+        $network = data_get($service, 'destination.network');
+
+        if (blank($network)) {
+            return [];
+        }
+
+        return [
+            'docker network connect '.escapeshellarg($network).' coolify-log-drain >/dev/null 2>&1 || true',
+        ];
+    }
+
+    private function shouldStopBeforeStarting(bool $pullLatestImages, bool $stopBeforeStart): bool
+    {
+        return $stopBeforeStart && ! $pullLatestImages;
+    }
 }

app/Actions/Service/StopService.php

@@ -3,10 +3,12 @@
 namespace App\Actions\Service;
 
 use App\Actions\Server\CleanupDocker;
+use App\Enums\ProcessStatus;
 use App\Events\ServiceStatusChanged;
 use App\Models\Server;
 use App\Models\Service;
 use Lorisleiva\Actions\Concerns\AsAction;
+use Spatie\Activitylog\Models\Activity;
 
 class StopService
 {
@@ -17,6 +19,17 @@ class StopService
     public function handle(Service $service, bool $deleteConnectedNetworks = false, bool $dockerCleanup = true)
     {
         try {
+            // Cancel any in-progress deployment activities so status doesn't stay stuck at "starting"
+            Activity::where('properties->type_uuid', $service->uuid)
+                ->where(function ($q) {
+                    $q->where('properties->status', ProcessStatus::IN_PROGRESS->value)
+                        ->orWhere('properties->status', ProcessStatus::QUEUED->value);
+                })
+                ->each(function ($activity) {
+                    $activity->properties = $activity->properties->put('status', ProcessStatus::CANCELLED->value);
+                    $activity->save();
+                });
+
             $server = $service->destination->server;
             if (! $server->isFunctional()) {
                 return 'Server is not functional';
@@ -54,7 +67,7 @@ class StopService
         $timeout = count($containersToStop) > 5 ? 10 : 30;
         $commands = [];
         $containerList = implode(' ', $containersToStop);
-        $commands[] = "docker stop --time=$timeout $containerList";
+        $commands[] = "docker stop -t $timeout $containerList";
         $commands[] = "docker rm -f $containerList";
         instant_remote_process(
             command: $commands,

app/Actions/Shared/ComplexStatusCheck.php

@@ -3,11 +3,14 @@
 namespace App\Actions\Shared;
 
 use App\Models\Application;
+use App\Services\ContainerStatusAggregator;
+use App\Traits\CalculatesExcludedStatus;
 use Lorisleiva\Actions\Concerns\AsAction;
 
 class ComplexStatusCheck
 {
     use AsAction;
+    use CalculatesExcludedStatus;
 
     public function handle(Application $application)
     {
@@ -17,44 +20,69 @@ class ComplexStatusCheck
             $is_main_server = $application->destination->server->id === $server->id;
             if (! $server->isFunctional()) {
                 if ($is_main_server) {
-                    $application->update(['status' => 'exited:unhealthy']);
+                    $application->update(['status' => 'exited']);
 
                     continue;
                 } else {
-                    $application->additional_servers()->updateExistingPivot($server->id, ['status' => 'exited:unhealthy']);
+                    $application->additional_servers()->updateExistingPivot($server->id, ['status' => 'exited']);
 
                     continue;
                 }
             }
-            $container = instant_remote_process(["docker container inspect $(docker container ls -q --filter 'label=coolify.applicationId={$application->id}' --filter 'label=coolify.pullRequestId=0') --format '{{json .}}'"], $server, false);
-            $container = format_docker_command_output_to_json($container);
-            if ($container->count() === 1) {
-                $container = $container->first();
-                $containerStatus = data_get($container, 'State.Status');
-                $containerHealth = data_get($container, 'State.Health.Status', 'unhealthy');
+            $containers = instant_remote_process(["docker container inspect $(docker container ls -q --filter 'label=coolify.applicationId={$application->id}' --filter 'label=coolify.pullRequestId=0') --format '{{json .}}'"], $server, false);
+            $containers = format_docker_command_output_to_json($containers);
+
+            if ($containers->count() > 0) {
+                $statusToSet = $this->aggregateContainerStatuses($application, $containers);
+
                 if ($is_main_server) {
                     $statusFromDb = $application->status;
-                    if ($statusFromDb !== $containerStatus) {
-                        $application->update(['status' => "$containerStatus:$containerHealth"]);
+                    if ($statusFromDb !== $statusToSet) {
+                        $application->update(['status' => $statusToSet]);
                     }
                 } else {
                     $additional_server = $application->additional_servers()->wherePivot('server_id', $server->id);
                     $statusFromDb = $additional_server->first()->pivot->status;
-                    if ($statusFromDb !== $containerStatus) {
-                        $additional_server->updateExistingPivot($server->id, ['status' => "$containerStatus:$containerHealth"]);
+                    if ($statusFromDb !== $statusToSet) {
+                        $additional_server->updateExistingPivot($server->id, ['status' => $statusToSet]);
                     }
                 }
             } else {
                 if ($is_main_server) {
-                    $application->update(['status' => 'exited:unhealthy']);
+                    $application->update(['status' => 'exited']);
 
                     continue;
                 } else {
-                    $application->additional_servers()->updateExistingPivot($server->id, ['status' => 'exited:unhealthy']);
+                    $application->additional_servers()->updateExistingPivot($server->id, ['status' => 'exited']);
 
                     continue;
                 }
             }
         }
     }
+
+    private function aggregateContainerStatuses($application, $containers)
+    {
+        $dockerComposeRaw = data_get($application, 'docker_compose_raw');
+        $excludedContainers = $this->getExcludedContainersFromDockerCompose($dockerComposeRaw);
+
+        // Filter non-excluded containers
+        $relevantContainers = collect($containers)->filter(function ($container) use ($excludedContainers) {
+            $labels = data_get($container, 'Config.Labels', []);
+            $serviceName = data_get($labels, 'com.docker.compose.service');
+
+            return ! ($serviceName && $excludedContainers->contains($serviceName));
+        });
+
+        // If all containers are excluded, calculate status from excluded containers
+        // but mark it with :excluded to indicate monitoring is disabled
+        if ($relevantContainers->isEmpty()) {
+            return $this->calculateExcludedStatus($containers, $excludedContainers);
+        }
+
+        // Use ContainerStatusAggregator service for state machine logic
+        $aggregator = new ContainerStatusAggregator;
+
+        return $aggregator->aggregateFromContainers($relevantContainers);
+    }
 }

app/Actions/Stripe/CancelSubscription.php

@@ -0,0 +1,210 @@
+<?php
+
+namespace App\Actions\Stripe;
+
+use App\Models\Subscription;
+use App\Models\User;
+use Illuminate\Support\Collection;
+use Stripe\Exception\InvalidRequestException;
+use Stripe\StripeClient;
+
+class CancelSubscription
+{
+    private User $user;
+
+    private bool $isDryRun;
+
+    private ?StripeClient $stripe = null;
+
+    public function __construct(User $user, bool $isDryRun = false)
+    {
+        $this->user = $user;
+        $this->isDryRun = $isDryRun;
+
+        if (! $isDryRun && isCloud()) {
+            $this->stripe = app(StripeClient::class);
+        }
+    }
+
+    public function getSubscriptionsPreview(): Collection
+    {
+        $subscriptions = collect();
+
+        // Get all teams the user belongs to
+        $teams = $this->user->teams()->get();
+
+        foreach ($teams as $team) {
+            // Only include subscriptions from teams where user is owner
+            $userRole = $team->pivot->role;
+            if ($userRole === 'owner' && $team->subscription) {
+                $subscription = $team->subscription;
+
+                // Only include active subscriptions
+                if ($subscription->stripe_subscription_id &&
+                    $subscription->stripe_invoice_paid) {
+                    $subscriptions->push($subscription);
+                }
+            }
+        }
+
+        return $subscriptions;
+    }
+
+    /**
+     * Verify subscriptions exist and are active in Stripe API
+     *
+     * @return array ['verified' => Collection, 'not_found' => Collection, 'errors' => array]
+     */
+    public function verifySubscriptionsInStripe(): array
+    {
+        if (! isCloud()) {
+            return [
+                'verified' => collect(),
+                'not_found' => collect(),
+                'errors' => [],
+            ];
+        }
+
+        $stripe = app(StripeClient::class);
+        $subscriptions = $this->getSubscriptionsPreview();
+
+        $verified = collect();
+        $notFound = collect();
+        $errors = [];
+
+        foreach ($subscriptions as $subscription) {
+            try {
+                $stripeSubscription = $stripe->subscriptions->retrieve($subscription->stripe_subscription_id);
+
+                // Check if subscription is actually active in Stripe
+                if (in_array($stripeSubscription->status, ['active', 'trialing', 'past_due'])) {
+                    $verified->push([
+                        'subscription' => $subscription,
+                        'stripe_status' => $stripeSubscription->status,
+                        'current_period_end' => $stripeSubscription->current_period_end,
+                    ]);
+                } else {
+                    $notFound->push([
+                        'subscription' => $subscription,
+                        'reason' => "Status in Stripe: {$stripeSubscription->status}",
+                    ]);
+                }
+            } catch (InvalidRequestException $e) {
+                // Subscription doesn't exist in Stripe
+                $notFound->push([
+                    'subscription' => $subscription,
+                    'reason' => 'Not found in Stripe',
+                ]);
+            } catch (\Exception $e) {
+                $errors[] = "Error verifying subscription {$subscription->stripe_subscription_id}: ".$e->getMessage();
+                \Log::error("Error verifying subscription {$subscription->stripe_subscription_id}: ".$e->getMessage());
+            }
+        }
+
+        return [
+            'verified' => $verified,
+            'not_found' => $notFound,
+            'errors' => $errors,
+        ];
+    }
+
+    public function execute(): array
+    {
+        if ($this->isDryRun) {
+            return [
+                'cancelled' => 0,
+                'failed' => 0,
+                'errors' => [],
+            ];
+        }
+
+        $cancelledCount = 0;
+        $failedCount = 0;
+        $errors = [];
+
+        $subscriptions = $this->getSubscriptionsPreview();
+
+        foreach ($subscriptions as $subscription) {
+            try {
+                $this->cancelSingleSubscription($subscription);
+                $cancelledCount++;
+            } catch (\Exception $e) {
+                $failedCount++;
+                $errorMessage = "Failed to cancel subscription {$subscription->stripe_subscription_id}: ".$e->getMessage();
+                $errors[] = $errorMessage;
+                \Log::error($errorMessage);
+            }
+        }
+
+        return [
+            'cancelled' => $cancelledCount,
+            'failed' => $failedCount,
+            'errors' => $errors,
+        ];
+    }
+
+    private function cancelSingleSubscription(Subscription $subscription): void
+    {
+        if (! $this->stripe) {
+            throw new \Exception('Stripe client not initialized');
+        }
+
+        $subscriptionId = $subscription->stripe_subscription_id;
+
+        // Cancel the subscription immediately (not at period end)
+        $this->stripe->subscriptions->cancel($subscriptionId, []);
+
+        // Update local database
+        $subscription->update([
+            'stripe_cancel_at_period_end' => false,
+            'stripe_invoice_paid' => false,
+            'stripe_trial_already_ended' => false,
+            'stripe_past_due' => false,
+            'stripe_feedback' => 'User account deleted',
+            'stripe_comment' => 'Subscription cancelled due to user account deletion at '.now()->toDateTimeString(),
+        ]);
+
+        // Call the team's subscription ended method to handle cleanup
+        if ($subscription->team) {
+            $subscription->team->subscriptionEnded();
+        }
+
+        \Log::info("Cancelled Stripe subscription: {$subscriptionId} for team: {$subscription->team->name}");
+    }
+
+    /**
+     * Cancel a single subscription by ID (helper method for external use)
+     */
+    public static function cancelById(string $subscriptionId): bool
+    {
+        try {
+            if (! isCloud()) {
+                return false;
+            }
+
+            $stripe = app(StripeClient::class);
+            $stripe->subscriptions->cancel($subscriptionId, []);
+
+            // Update local record if exists
+            $subscription = Subscription::where('stripe_subscription_id', $subscriptionId)->first();
+            if ($subscription) {
+                $subscription->update([
+                    'stripe_cancel_at_period_end' => false,
+                    'stripe_invoice_paid' => false,
+                    'stripe_trial_already_ended' => false,
+                    'stripe_past_due' => false,
+                ]);
+
+                if ($subscription->team) {
+                    $subscription->team->subscriptionEnded();
+                }
+            }
+
+            return true;
+        } catch (\Exception $e) {
+            \Log::error("Failed to cancel subscription {$subscriptionId}: ".$e->getMessage());
+
+            return false;
+        }
+    }
+}

app/Actions/Stripe/CancelSubscriptionAtPeriodEnd.php

@@ -0,0 +1,61 @@
+<?php
+
+namespace App\Actions\Stripe;
+
+use App\Models\Team;
+use Stripe\Exception\InvalidRequestException;
+use Stripe\StripeClient;
+
+class CancelSubscriptionAtPeriodEnd
+{
+    private StripeClient $stripe;
+
+    public function __construct(?StripeClient $stripe = null)
+    {
+        $this->stripe = $stripe ?? app(StripeClient::class);
+    }
+
+    /**
+     * Cancel the team's subscription at the end of the current billing period.
+     *
+     * @return array{success: bool, error: string|null}
+     */
+    public function execute(Team $team): array
+    {
+        $subscription = $team->subscription;
+
+        if (! $subscription?->stripe_subscription_id) {
+            return ['success' => false, 'error' => 'No active subscription found.'];
+        }
+
+        if (! $subscription->stripe_invoice_paid) {
+            return ['success' => false, 'error' => 'Subscription is not active.'];
+        }
+
+        if ($subscription->stripe_cancel_at_period_end) {
+            return ['success' => false, 'error' => 'Subscription is already set to cancel at the end of the billing period.'];
+        }
+
+        try {
+            $this->stripe->subscriptions->update($subscription->stripe_subscription_id, [
+                'cancel_at_period_end' => true,
+            ]);
+
+            $subscription->update([
+                'stripe_cancel_at_period_end' => true,
+            ]);
+
+            \Log::info("Subscription {$subscription->stripe_subscription_id} set to cancel at period end for team {$team->name}");
+
+            return ['success' => true, 'error' => null];
+        } catch (InvalidRequestException $e) {
+            \Log::error("Stripe cancel at period end error for team {$team->id}: ".$e->getMessage());
+
+            return ['success' => false, 'error' => 'Stripe error: '.$e->getMessage()];
+        } catch (\Exception $e) {
+            \Log::error("Cancel at period end error for team {$team->id}: ".$e->getMessage());
+
+            return ['success' => false, 'error' => 'An unexpected error occurred. Please contact support.'];
+        }
+    }
+}

app/Actions/Stripe/RefundSubscription.php

@@ -0,0 +1,158 @@
+<?php
+
+namespace App\Actions\Stripe;
+
+use App\Models\Team;
+use Carbon\Carbon;
+use Stripe\Exception\InvalidRequestException;
+use Stripe\StripeClient;
+
+class RefundSubscription
+{
+    private StripeClient $stripe;
+
+    private const REFUND_WINDOW_DAYS = 30;
+
+    public function __construct(?StripeClient $stripe = null)
+    {
+        $this->stripe = $stripe ?? app(StripeClient::class);
+    }
+
+    /**
+     * Check if the team's subscription is eligible for a refund.
+     *
+     * @return array{eligible: bool, days_remaining: int, reason: string, current_period_end: int|null}
+     */
+    public function checkEligibility(Team $team): array
+    {
+        $subscription = $team->subscription;
+
+        if ($subscription?->stripe_refunded_at) {
+            return $this->ineligible('A refund has already been processed for this team.');
+        }
+
+        if (! $subscription?->stripe_subscription_id) {
+            return $this->ineligible('No active subscription found.');
+        }
+
+        if (! $subscription->stripe_invoice_paid) {
+            return $this->ineligible('Subscription invoice is not paid.');
+        }
+
+        try {
+            $stripeSubscription = $this->stripe->subscriptions->retrieve($subscription->stripe_subscription_id);
+        } catch (InvalidRequestException $e) {
+            return $this->ineligible('Subscription not found in Stripe.');
+        }
+
+        $currentPeriodEnd = $stripeSubscription->current_period_end;
+
+        if (! in_array($stripeSubscription->status, ['active', 'trialing'])) {
+            return $this->ineligible("Subscription status is '{$stripeSubscription->status}'.", $currentPeriodEnd);
+        }
+
+        $startDate = Carbon::createFromTimestamp($stripeSubscription->start_date);
+        $daysSinceStart = (int) $startDate->diffInDays(now());
+        $daysRemaining = self::REFUND_WINDOW_DAYS - $daysSinceStart;
+
+        if ($daysRemaining <= 0) {
+            return $this->ineligible('The 30-day refund window has expired.', $currentPeriodEnd);
+        }
+
+        return [
+            'eligible' => true,
+            'days_remaining' => $daysRemaining,
+            'reason' => 'Eligible for refund.',
+            'current_period_end' => $currentPeriodEnd,
+        ];
+    }
+
+    /**
+     * Process a full refund and cancel the subscription.
+     *
+     * @return array{success: bool, error: string|null}
+     */
+    public function execute(Team $team): array
+    {
+        $eligibility = $this->checkEligibility($team);
+
+        if (! $eligibility['eligible']) {
+            return ['success' => false, 'error' => $eligibility['reason']];
+        }
+
+        $subscription = $team->subscription;
+
+        try {
+            $invoices = $this->stripe->invoices->all([
+                'subscription' => $subscription->stripe_subscription_id,
+                'status' => 'paid',
+                'limit' => 1,
+            ]);
+
+            if (empty($invoices->data)) {
+                return ['success' => false, 'error' => 'No paid invoice found to refund.'];
+            }
+
+            $invoice = $invoices->data[0];
+            $paymentIntentId = $invoice->payment_intent;
+
+            if (! $paymentIntentId) {
+                return ['success' => false, 'error' => 'No payment intent found on the invoice.'];
+            }
+
+            $this->stripe->refunds->create([
+                'payment_intent' => $paymentIntentId,
+            ]);
+
+            // Record refund immediately so it cannot be retried if cancel fails
+            $subscription->update([
+                'stripe_refunded_at' => now(),
+                'stripe_feedback' => 'Refund requested by user',
+                'stripe_comment' => 'Full refund processed within 30-day window at '.now()->toDateTimeString(),
+            ]);
+
+            try {
+                $this->stripe->subscriptions->cancel($subscription->stripe_subscription_id);
+            } catch (\Exception $e) {
+                \Log::critical("Refund succeeded but subscription cancel failed for team {$team->id}: ".$e->getMessage());
+                send_internal_notification(
+                    "CRITICAL: Refund succeeded but cancel failed for subscription {$subscription->stripe_subscription_id}, team {$team->id}. Manual intervention required."
+                );
+            }
+
+            $subscription->update([
+                'stripe_cancel_at_period_end' => false,
+                'stripe_invoice_paid' => false,
+                'stripe_trial_already_ended' => false,
+                'stripe_past_due' => false,
+            ]);
+
+            $team->subscriptionEnded();
+
+            \Log::info("Refunded and cancelled subscription {$subscription->stripe_subscription_id} for team {$team->name}");
+
+            return ['success' => true, 'error' => null];
+        } catch (InvalidRequestException $e) {
+            \Log::error("Stripe refund error for team {$team->id}: ".$e->getMessage());
+
+            return ['success' => false, 'error' => 'Stripe error: '.$e->getMessage()];
+        } catch (\Exception $e) {
+            \Log::error("Refund error for team {$team->id}: ".$e->getMessage());
+
+            return ['success' => false, 'error' => 'An unexpected error occurred. Please contact support.'];
+        }
+    }
+
+    /**
+     * @return array{eligible: bool, days_remaining: int, reason: string, current_period_end: int|null}
+     */
+    private function ineligible(string $reason, ?int $currentPeriodEnd = null): array
+    {
+        return [
+            'eligible' => false,
+            'days_remaining' => 0,
+            'reason' => $reason,
+            'current_period_end' => $currentPeriodEnd,
+        ];
+    }
+}

app/Actions/Stripe/ResumeSubscription.php

@@ -0,0 +1,57 @@
+<?php
+
+namespace App\Actions\Stripe;
+
+use App\Models\Team;
+use Stripe\Exception\InvalidRequestException;
+use Stripe\StripeClient;
+
+class ResumeSubscription
+{
+    private StripeClient $stripe;
+
+    public function __construct(?StripeClient $stripe = null)
+    {
+        $this->stripe = $stripe ?? app(StripeClient::class);
+    }
+
+    /**
+     * Resume a subscription that was set to cancel at the end of the billing period.
+     *
+     * @return array{success: bool, error: string|null}
+     */
+    public function execute(Team $team): array
+    {
+        $subscription = $team->subscription;
+
+        if (! $subscription?->stripe_subscription_id) {
+            return ['success' => false, 'error' => 'No active subscription found.'];
+        }
+
+        if (! $subscription->stripe_cancel_at_period_end) {
+            return ['success' => false, 'error' => 'Subscription is not set to cancel.'];
+        }
+
+        try {
+            $this->stripe->subscriptions->update($subscription->stripe_subscription_id, [
+                'cancel_at_period_end' => false,
+            ]);
+
+            $subscription->update([
+                'stripe_cancel_at_period_end' => false,
+            ]);
+
+            \Log::info("Subscription {$subscription->stripe_subscription_id} resumed for team {$team->name}");
+
+            return ['success' => true, 'error' => null];
+        } catch (InvalidRequestException $e) {
+            \Log::error("Stripe resume subscription error for team {$team->id}: ".$e->getMessage());
+
+            return ['success' => false, 'error' => 'Stripe error: '.$e->getMessage()];
+        } catch (\Exception $e) {
+            \Log::error("Resume subscription error for team {$team->id}: ".$e->getMessage());
+
+            return ['success' => false, 'error' => 'An unexpected error occurred. Please contact support.'];
+        }
+    }
+}

app/Actions/Stripe/UpdateSubscriptionQuantity.php

@@ -0,0 +1,207 @@
+<?php
+
+namespace App\Actions\Stripe;
+
+use App\Jobs\ServerLimitCheckJob;
+use App\Models\Team;
+use Stripe\Exception\InvalidRequestException;
+use Stripe\StripeClient;
+
+class UpdateSubscriptionQuantity
+{
+    public const int MAX_SERVER_LIMIT = 100;
+
+    public const int MIN_SERVER_LIMIT = 2;
+
+    private StripeClient $stripe;
+
+    public function __construct(?StripeClient $stripe = null)
+    {
+        $this->stripe = $stripe ?? app(StripeClient::class);
+    }
+
+    /**
+     * Fetch a full price preview for a quantity change from Stripe.
+     * Returns both the prorated amount due now and the recurring cost for the next billing cycle.
+     *
+     * @return array{success: bool, error: string|null, preview: array{due_now: int, recurring_subtotal: int, recurring_tax: int, recurring_total: int, unit_price: int, tax_description: string|null, quantity: int, currency: string}|null}
+     */
+    public function fetchPricePreview(Team $team, int $quantity): array
+    {
+        $subscription = $team->subscription;
+
+        if (! $subscription?->stripe_subscription_id || ! $subscription->stripe_invoice_paid) {
+            return ['success' => false, 'error' => 'No active subscription found.', 'preview' => null];
+        }
+
+        try {
+            $stripeSubscription = $this->stripe->subscriptions->retrieve($subscription->stripe_subscription_id);
+            $item = $stripeSubscription->items->data[0] ?? null;
+
+            if (! $item) {
+                return ['success' => false, 'error' => 'Could not retrieve subscription details.', 'preview' => null];
+            }
+
+            $currency = strtoupper($item->price->currency ?? 'usd');
+            $billingInterval = $item->price->recurring->interval ?? 'month';
+
+            // Upcoming invoice gives us the prorated amount due now
+            $upcomingInvoice = $this->stripe->invoices->upcoming([
+                'customer' => $subscription->stripe_customer_id,
+                'subscription' => $subscription->stripe_subscription_id,
+                'subscription_items' => [
+                    ['id' => $item->id, 'quantity' => $quantity],
+                ],
+                'subscription_proration_behavior' => 'create_prorations',
+            ]);
+
+            // Extract tax percentage — try total_tax_amounts first, fall back to invoice tax/subtotal
+            $taxPercentage = 0.0;
+            $taxDescription = null;
+            if (! empty($upcomingInvoice->total_tax_amounts)) {
+                $taxAmount = $upcomingInvoice->total_tax_amounts[0] ?? null;
+                if ($taxAmount?->tax_rate) {
+                    $taxRate = $this->stripe->taxRates->retrieve($taxAmount->tax_rate);
+                    $taxPercentage = (float) ($taxRate->percentage ?? 0);
+                    $taxDescription = $taxRate->display_name.' ('.$taxRate->jurisdiction.') '.$taxRate->percentage.'%';
+                }
+            }
+            // Fallback tax percentage from invoice totals - use tax_rate details when available for accuracy
+            if ($taxPercentage === 0.0 && ($upcomingInvoice->tax ?? 0) > 0 && ($upcomingInvoice->subtotal ?? 0) > 0) {
+                $taxPercentage = round(($upcomingInvoice->tax / $upcomingInvoice->subtotal) * 100, 2);
+            }
+
+            // Recurring cost for next cycle — read from non-proration invoice lines
+            $recurringSubtotal = 0;
+            foreach ($upcomingInvoice->lines->data as $line) {
+                if (! $line->proration) {
+                    $recurringSubtotal += $line->amount;
+                }
+            }
+            $unitPrice = $quantity > 0 ? (int) round($recurringSubtotal / $quantity) : 0;
+
+            $recurringTax = $taxPercentage > 0
+                ? (int) round($recurringSubtotal * $taxPercentage / 100)
+                : 0;
+            $recurringTotal = $recurringSubtotal + $recurringTax;
+
+            // Due now = amount_due (accounts for customer balance/credits) minus recurring
+            $amountDue = $upcomingInvoice->amount_due ?? $upcomingInvoice->total ?? 0;
+            $dueNow = $amountDue - $recurringTotal;
+
+            return [
+                'success' => true,
+                'error' => null,
+                'preview' => [
+                    'due_now' => $dueNow,
+                    'recurring_subtotal' => $recurringSubtotal,
+                    'recurring_tax' => $recurringTax,
+                    'recurring_total' => $recurringTotal,
+                    'unit_price' => $unitPrice,
+                    'tax_description' => $taxDescription,
+                    'quantity' => $quantity,
+                    'currency' => $currency,
+                    'billing_interval' => $billingInterval,
+                ],
+            ];
+        } catch (\Exception $e) {
+            \Log::warning("Stripe fetch price preview error for team {$team->id}: ".$e->getMessage());
+
+            return ['success' => false, 'error' => 'Could not load price preview.', 'preview' => null];
+        }
+    }
+
+    /**
+     * Update the subscription quantity (server limit) for a team.
+     *
+     * @return array{success: bool, error: string|null}
+     */
+    public function execute(Team $team, int $quantity): array
+    {
+        if ($quantity < self::MIN_SERVER_LIMIT) {
+            return ['success' => false, 'error' => 'Minimum server limit is '.self::MIN_SERVER_LIMIT.'.'];
+        }
+
+        $subscription = $team->subscription;
+
+        if (! $subscription?->stripe_subscription_id) {
+            return ['success' => false, 'error' => 'No active subscription found.'];
+        }
+
+        if (! $subscription->stripe_invoice_paid) {
+            return ['success' => false, 'error' => 'Subscription is not active.'];
+        }
+
+        try {
+            $stripeSubscription = $this->stripe->subscriptions->retrieve($subscription->stripe_subscription_id);
+            $item = $stripeSubscription->items->data[0] ?? null;
+
+            if (! $item?->id) {
+                return ['success' => false, 'error' => 'Could not find subscription item.'];
+            }
+
+            $previousQuantity = $item->quantity ?? $team->custom_server_limit;
+
+            $updatedSubscription = $this->stripe->subscriptions->update($subscription->stripe_subscription_id, [
+                'items' => [
+                    ['id' => $item->id, 'quantity' => $quantity],
+                ],
+                'proration_behavior' => 'always_invoice',
+                'expand' => ['latest_invoice'],
+            ]);
+
+            // Check if the proration invoice was paid
+            $latestInvoice = $updatedSubscription->latest_invoice;
+            if ($latestInvoice && $latestInvoice->status !== 'paid') {
+                \Log::warning("Subscription {$subscription->stripe_subscription_id} quantity updated but invoice not paid (status: {$latestInvoice->status}) for team {$team->name}. Reverting to {$previousQuantity}.");
+
+                // Revert subscription quantity on Stripe
+                try {
+                    $this->stripe->subscriptions->update($subscription->stripe_subscription_id, [
+                        'items' => [
+                            ['id' => $item->id, 'quantity' => $previousQuantity],
+                        ],
+                        'proration_behavior' => 'none',
+                    ]);
+                } catch (\Exception $revertException) {
+                    \Log::critical("Failed to revert Stripe quantity for subscription {$subscription->stripe_subscription_id}, team {$team->id}. Stripe may have quantity {$quantity} but local is {$previousQuantity}. Error: ".$revertException->getMessage());
+                    send_internal_notification(
+                        "CRITICAL: Stripe quantity revert failed for subscription {$subscription->stripe_subscription_id}, team {$team->id}. Manual reconciliation required."
+                    );
+                }
+
+                // Void the unpaid invoice
+                if ($latestInvoice->id) {
+                    $this->stripe->invoices->voidInvoice($latestInvoice->id);
+                }
+
+                return ['success' => false, 'error' => 'Payment failed. Your server limit was not changed. Please check your payment method and try again.'];
+            }
+
+            $team->update([
+                'custom_server_limit' => $quantity,
+            ]);
+
+            ServerLimitCheckJob::dispatch($team);
+
+            \Log::info("Subscription {$subscription->stripe_subscription_id} quantity updated to {$quantity} for team {$team->name}");
+
+            return ['success' => true, 'error' => null];
+        } catch (InvalidRequestException $e) {
+            \Log::error("Stripe update quantity error for team {$team->id}: ".$e->getMessage());
+
+            return ['success' => false, 'error' => 'Stripe error: '.$e->getMessage()];
+        } catch (\Exception $e) {
+            \Log::error("Update subscription quantity error for team {$team->id}: ".$e->getMessage());
+
+            return ['success' => false, 'error' => 'An unexpected error occurred. Please contact support.'];
+        }
+    }
+
+    private function formatAmount(int $cents, string $currency): string
+    {
+        return strtoupper($currency) === 'USD'
+            ? '$'.number_format($cents / 100, 2)
+            : number_format($cents / 100, 2).' '.$currency;
+    }
+}

app/Actions/User/DeleteUserResources.php

@@ -0,0 +1,131 @@
+<?php
+
+namespace App\Actions\User;
+
+use App\Models\User;
+use Illuminate\Support\Collection;
+
+class DeleteUserResources
+{
+    private User $user;
+
+    private bool $isDryRun;
+
+    public function __construct(User $user, bool $isDryRun = false)
+    {
+        $this->user = $user;
+        $this->isDryRun = $isDryRun;
+    }
+
+    public function getResourcesPreview(): array
+    {
+        $applications = collect();
+        $databases = collect();
+        $services = collect();
+
+        // Get all teams the user belongs to
+        $teams = $this->user->teams()->get();
+
+        foreach ($teams as $team) {
+            // Only delete resources from teams that will be FULLY DELETED
+            // This means: user is the ONLY member of the team
+            //
+            // DO NOT delete resources if:
+            // - User is just a member (not owner)
+            // - Team has other members (ownership will be transferred or user just removed)
+
+            $userRole = $team->pivot->role;
+            $memberCount = $team->members->count();
+
+            // Skip if user is not owner
+            if ($userRole !== 'owner') {
+                continue;
+            }
+
+            // Skip if team has other members (will be transferred/user removed, not deleted)
+            if ($memberCount > 1) {
+                continue;
+            }
+
+            // Only delete resources from teams where user is the ONLY member
+            // These teams will be fully deleted
+
+            // Get all servers for this team
+            $servers = $team->servers()->get();
+
+            foreach ($servers as $server) {
+                // Get applications (custom method returns Collection)
+                $serverApplications = $server->applications();
+                $applications = $applications->merge($serverApplications);
+
+                // Get databases (custom method returns Collection)
+                $serverDatabases = $server->databases();
+                $databases = $databases->merge($serverDatabases);
+
+                // Get services (relationship needs ->get())
+                $serverServices = $server->services()->get();
+                $services = $services->merge($serverServices);
+            }
+        }
+
+        return [
+            'applications' => $applications->unique('id'),
+            'databases' => $databases->unique('id'),
+            'services' => $services->unique('id'),
+        ];
+    }
+
+    public function execute(): array
+    {
+        if ($this->isDryRun) {
+            return [
+                'applications' => 0,
+                'databases' => 0,
+                'services' => 0,
+            ];
+        }
+
+        $deletedCounts = [
+            'applications' => 0,
+            'databases' => 0,
+            'services' => 0,
+        ];
+
+        $resources = $this->getResourcesPreview();
+
+        // Delete applications
+        foreach ($resources['applications'] as $application) {
+            try {
+                $application->forceDelete();
+                $deletedCounts['applications']++;
+            } catch (\Exception $e) {
+                \Log::error("Failed to delete application {$application->id}: ".$e->getMessage());
+                throw $e; // Re-throw to trigger rollback
+            }
+        }
+
+        // Delete databases
+        foreach ($resources['databases'] as $database) {
+            try {
+                $database->forceDelete();
+                $deletedCounts['databases']++;
+            } catch (\Exception $e) {
+                \Log::error("Failed to delete database {$database->id}: ".$e->getMessage());
+                throw $e; // Re-throw to trigger rollback
+            }
+        }
+
+        // Delete services
+        foreach ($resources['services'] as $service) {
+            try {
+                $service->forceDelete();
+                $deletedCounts['services']++;
+            } catch (\Exception $e) {
+                \Log::error("Failed to delete service {$service->id}: ".$e->getMessage());
+                throw $e; // Re-throw to trigger rollback
+            }
+        }
+
+        return $deletedCounts;
+    }
+}

app/Actions/User/DeleteUserServers.php

@@ -0,0 +1,77 @@
+<?php
+
+namespace App\Actions\User;
+
+use App\Models\Server;
+use App\Models\User;
+use Illuminate\Support\Collection;
+
+class DeleteUserServers
+{
+    private User $user;
+
+    private bool $isDryRun;
+
+    public function __construct(User $user, bool $isDryRun = false)
+    {
+        $this->user = $user;
+        $this->isDryRun = $isDryRun;
+    }
+
+    public function getServersPreview(): Collection
+    {
+        $servers = collect();
+
+        // Get all teams the user belongs to
+        $teams = $this->user->teams()->get();
+
+        foreach ($teams as $team) {
+            // Only include servers from teams where user is owner or admin
+            $userRole = $team->pivot->role;
+            if ($userRole === 'owner' || $userRole === 'admin') {
+                $teamServers = $team->servers()->get();
+                $servers = $servers->merge($teamServers);
+            }
+        }
+
+        // Return unique servers (in case same server is in multiple teams)
+        return $servers->unique('id');
+    }
+
+    public function execute(): array
+    {
+        if ($this->isDryRun) {
+            return [
+                'servers' => 0,
+            ];
+        }
+
+        $deletedCount = 0;
+
+        $servers = $this->getServersPreview();
+
+        foreach ($servers as $server) {
+            try {
+                // Skip the default server (ID 0) which is the Coolify host
+                if ($server->id === 0) {
+                    \Log::info('Skipping deletion of Coolify host server (ID: 0)');
+
+                    continue;
+                }
+
+                // The Server model's forceDeleting event will handle cleanup of:
+                // - destinations
+                // - settings
+                $server->forceDelete();
+                $deletedCount++;
+            } catch (\Exception $e) {
+                \Log::error("Failed to delete server {$server->id}: ".$e->getMessage());
+                throw $e; // Re-throw to trigger rollback
+            }
+        }
+
+        return [
+            'servers' => $deletedCount,
+        ];
+    }
+}

app/Actions/User/DeleteUserTeams.php

@@ -0,0 +1,205 @@
+<?php
+
+namespace App\Actions\User;
+
+use App\Models\Team;
+use App\Models\User;
+
+class DeleteUserTeams
+{
+    private User $user;
+
+    private bool $isDryRun;
+
+    public function __construct(User $user, bool $isDryRun = false)
+    {
+        $this->user = $user;
+        $this->isDryRun = $isDryRun;
+    }
+
+    public function getTeamsPreview(): array
+    {
+        $teamsToDelete = collect();
+        $teamsToTransfer = collect();
+        $teamsToLeave = collect();
+        $edgeCases = collect();
+
+        $teams = $this->user->teams;
+
+        foreach ($teams as $team) {
+            // Skip root team (ID 0)
+            if ($team->id === 0) {
+                continue;
+            }
+
+            $userRole = $team->pivot->role;
+            $memberCount = $team->members->count();
+
+            if ($memberCount === 1) {
+                // User is alone in the team - delete it
+                $teamsToDelete->push($team);
+            } elseif ($userRole === 'owner') {
+                // Check if there are other owners
+                $otherOwners = $team->members
+                    ->where('id', '!=', $this->user->id)
+                    ->filter(function ($member) {
+                        return $member->pivot->role === 'owner';
+                    });
+
+                if ($otherOwners->isNotEmpty()) {
+                    // There are other owners, but check if this user is paying for the subscription
+                    if ($this->isUserPayingForTeamSubscription($team)) {
+                        // User is paying for the subscription - this is an edge case
+                        $edgeCases->push([
+                            'team' => $team,
+                            'reason' => 'User is paying for the team\'s Stripe subscription but there are other owners. The subscription needs to be cancelled or transferred to another owner\'s payment method.',
+                        ]);
+                    } else {
+                        // There are other owners and user is not paying, just remove this user
+                        $teamsToLeave->push($team);
+                    }
+                } else {
+                    // User is the only owner, check for replacement
+                    $newOwner = $this->findNewOwner($team);
+                    if ($newOwner) {
+                        $teamsToTransfer->push([
+                            'team' => $team,
+                            'new_owner' => $newOwner,
+                        ]);
+                    } else {
+                        // No suitable replacement found - this is an edge case
+                        $edgeCases->push([
+                            'team' => $team,
+                            'reason' => 'No suitable owner replacement found. Team has only regular members without admin privileges.',
+                        ]);
+                    }
+                }
+            } else {
+                // User is just a member - remove them from the team
+                $teamsToLeave->push($team);
+            }
+        }
+
+        return [
+            'to_delete' => $teamsToDelete,
+            'to_transfer' => $teamsToTransfer,
+            'to_leave' => $teamsToLeave,
+            'edge_cases' => $edgeCases,
+        ];
+    }
+
+    public function execute(): array
+    {
+        if ($this->isDryRun) {
+            return [
+                'deleted' => 0,
+                'transferred' => 0,
+                'left' => 0,
+            ];
+        }
+
+        $counts = [
+            'deleted' => 0,
+            'transferred' => 0,
+            'left' => 0,
+        ];
+
+        $preview = $this->getTeamsPreview();
+
+        // Check for edge cases - should not happen here as we check earlier, but be safe
+        if ($preview['edge_cases']->isNotEmpty()) {
+            throw new \Exception('Edge cases detected during execution. This should not happen.');
+        }
+
+        // Delete teams where user is alone
+        foreach ($preview['to_delete'] as $team) {
+            try {
+                // The Team model's deleting event will handle cleanup of:
+                // - private keys
+                // - sources
+                // - tags
+                // - environment variables
+                // - s3 storages
+                // - notification settings
+                $team->delete();
+                $counts['deleted']++;
+            } catch (\Exception $e) {
+                \Log::error("Failed to delete team {$team->id}: ".$e->getMessage());
+                throw $e; // Re-throw to trigger rollback
+            }
+        }
+
+        // Transfer ownership for teams where user is owner but not alone
+        foreach ($preview['to_transfer'] as $item) {
+            try {
+                $team = $item['team'];
+                $newOwner = $item['new_owner'];
+
+                // Update the new owner's role to owner
+                $team->members()->updateExistingPivot($newOwner->id, ['role' => 'owner']);
+                RevokeUserTeamTokens::forUserTeam($newOwner, $team->id);
+
+                // Remove the current user from the team
+                $team->members()->detach($this->user->id);
+                RevokeUserTeamTokens::forUserTeam($this->user, $team->id);
+
+                $counts['transferred']++;
+            } catch (\Exception $e) {
+                \Log::error("Failed to transfer ownership of team {$item['team']->id}: ".$e->getMessage());
+                throw $e; // Re-throw to trigger rollback
+            }
+        }
+
+        // Remove user from teams where they're just a member
+        foreach ($preview['to_leave'] as $team) {
+            try {
+                $team->members()->detach($this->user->id);
+                RevokeUserTeamTokens::forUserTeam($this->user, $team->id);
+                $counts['left']++;
+            } catch (\Exception $e) {
+                \Log::error("Failed to remove user from team {$team->id}: ".$e->getMessage());
+                throw $e; // Re-throw to trigger rollback
+            }
+        }
+
+        return $counts;
+    }
+
+    private function findNewOwner(Team $team): ?User
+    {
+        // Only look for admins as potential new owners
+        // We don't promote regular members automatically
+        $otherAdmin = $team->members
+            ->where('id', '!=', $this->user->id)
+            ->filter(function ($member) {
+                return $member->pivot->role === 'admin';
+            })
+            ->first();
+
+        return $otherAdmin;
+    }
+
+    private function isUserPayingForTeamSubscription(Team $team): bool
+    {
+        if (! $team->subscription || ! $team->subscription->stripe_customer_id) {
+            return false;
+        }
+
+        // In Stripe, we need to check if the customer email matches the user's email
+        // This would require a Stripe API call to get customer details
+        // For now, we'll check if the subscription was created by this user
+
+        // Alternative approach: Check if user is the one who initiated the subscription
+        // We could store this information when the subscription is created
+        // For safety, we'll assume if there's an active subscription and multiple owners,
+        // we should treat it as an edge case that needs manual review
+
+        if ($team->subscription->stripe_subscription_id &&
+            $team->subscription->stripe_invoice_paid) {
+            // Active subscription exists - we should be cautious
+            return true;
+        }
+
+        return false;
+    }
+}

app/Actions/User/RevokeUserTeamTokens.php

@@ -0,0 +1,43 @@
+<?php
+
+namespace App\Actions\User;
+
+use App\Models\PersonalAccessToken;
+use App\Models\User;
+use Illuminate\Database\Eloquent\Builder;
+
+class RevokeUserTeamTokens
+{
+    public static function forUserTeam(User|int $user, int|string $teamId): int
+    {
+        return self::baseQuery()
+            ->where('tokenable_id', self::userId($user))
+            ->where('team_id', $teamId)
+            ->delete();
+    }
+
+    public static function forUser(User|int $user): int
+    {
+        return self::baseQuery()
+            ->where('tokenable_id', self::userId($user))
+            ->delete();
+    }
+
+    public static function forTeam(int|string $teamId): int
+    {
+        return self::baseQuery()
+            ->where('team_id', $teamId)
+            ->delete();
+    }
+
+    private static function baseQuery(): Builder
+    {
+        return PersonalAccessToken::query()
+            ->where('tokenable_type', User::class);
+    }
+
+    private static function userId(User|int $user): int
+    {
+        return $user instanceof User ? $user->id : $user;
+    }
+}

app/Casts/EncryptedArrayCast.php

@@ -0,0 +1,51 @@
+<?php
+
+namespace App\Casts;
+
+use Illuminate\Contracts\Database\Eloquent\CastsAttributes;
+use Illuminate\Contracts\Encryption\DecryptException;
+use Illuminate\Database\Eloquent\Model;
+use Illuminate\Support\Facades\Crypt;
+
+/**
+ * Stores an array as an encrypted JSON string at rest. Tolerates legacy
+ * plaintext JSON rows written before the column was encrypted, so existing
+ * snapshots keep decoding instead of throwing.
+ *
+ * @implements CastsAttributes<array<mixed>|null, array<mixed>|null>
+ */
+class EncryptedArrayCast implements CastsAttributes
+{
+    /**
+     * @param  array<string, mixed>  $attributes
+     * @return array<mixed>|null
+     */
+    public function get(Model $model, string $key, mixed $value, array $attributes): ?array
+    {
+        if ($value === null || $value === '') {
+            return null;
+        }
+
+        try {
+            $value = Crypt::decryptString($value);
+        } catch (DecryptException) {
+            // Legacy plaintext JSON written before this column was encrypted.
+        }
+
+        $decoded = json_decode((string) $value, true);
+
+        return is_array($decoded) ? $decoded : null;
+    }
+
+    /**
+     * @param  array<string, mixed>  $attributes
+     */
+    public function set(Model $model, string $key, mixed $value, array $attributes): ?string
+    {
+        if ($value === null) {
+            return null;
+        }
+
+        return Crypt::encryptString(json_encode($value, JSON_THROW_ON_ERROR));
+    }
+}

app/Console/Commands/AdminRemoveUser.php

@@ -1,56 +0,0 @@
-<?php
-
-namespace App\Console\Commands;
-
-use App\Models\User;
-use Illuminate\Console\Command;
-
-class AdminRemoveUser extends Command
-{
-    /**
-     * The name and signature of the console command.
-     *
-     * @var string
-     */
-    protected $signature = 'admin:remove-user {email}';
-
-    /**
-     * The console command description.
-     *
-     * @var string
-     */
-    protected $description = 'Remove User from database';
-
-    /**
-     * Execute the console command.
-     */
-    public function handle()
-    {
-        try {
-            $email = $this->argument('email');
-            $confirm = $this->confirm('Are you sure you want to remove user with email: '.$email.'?');
-            if (! $confirm) {
-                $this->info('User removal cancelled.');
-
-                return;
-            }
-            $this->info("Removing user with email: $email");
-            $user = User::whereEmail($email)->firstOrFail();
-            $teams = $user->teams;
-            foreach ($teams as $team) {
-                if ($team->members->count() > 1) {
-                    $this->error('User is a member of a team with more than one member. Please remove user from team first.');
-
-                    return;
-                }
-                $team->delete();
-            }
-            $user->delete();
-        } catch (\Exception $e) {
-            $this->error('Failed to remove user.');
-            $this->error($e->getMessage());
-
-            return;
-        }
-    }
-}

app/Console/Commands/CheckTraefikVersionCommand.php

@@ -0,0 +1,30 @@
+<?php
+
+namespace App\Console\Commands;
+
+use App\Jobs\CheckTraefikVersionJob;
+use Illuminate\Console\Command;
+
+class CheckTraefikVersionCommand extends Command
+{
+    protected $signature = 'traefik:check-version';
+
+    protected $description = 'Check Traefik proxy versions on all servers and send notifications for outdated versions';
+
+    public function handle(): int
+    {
+        $this->info('Checking Traefik versions on all servers...');
+
+        try {
+            CheckTraefikVersionJob::dispatch();
+            $this->info('Traefik version check job dispatched successfully.');
+            $this->info('Notifications will be sent to teams with outdated Traefik versions.');
+
+            return Command::SUCCESS;
+        } catch (\Exception $e) {
+            $this->error('Failed to dispatch Traefik version check job: '.$e->getMessage());
+
+            return Command::FAILURE;
+        }
+    }
+}

app/Console/Commands/CleanupDatabase.php

@@ -64,13 +64,5 @@ class CleanupDatabase extends Command
         if ($this->option('yes')) {
             $scheduled_task_executions->delete();
         }
-
-        // Cleanup webhooks table
-        $webhooks = DB::table('webhooks')->where('created_at', '<', now()->subDays($keep_days));
-        $count = $webhooks->count();
-        echo "Delete $count entries from webhooks.\n";
-        if ($this->option('yes')) {
-            $webhooks->delete();
-        }
     }
 }

app/Console/Commands/CleanupNames.php

@@ -0,0 +1,214 @@
+<?php
+
+namespace App\Console\Commands;
+
+use App\Models\Application;
+use App\Models\Environment;
+use App\Models\PrivateKey;
+use App\Models\Project;
+use App\Models\S3Storage;
+use App\Models\ScheduledTask;
+use App\Models\Server;
+use App\Models\Service;
+use App\Models\StandaloneClickhouse;
+use App\Models\StandaloneDragonfly;
+use App\Models\StandaloneKeydb;
+use App\Models\StandaloneMariadb;
+use App\Models\StandaloneMongodb;
+use App\Models\StandaloneMysql;
+use App\Models\StandalonePostgresql;
+use App\Models\StandaloneRedis;
+use App\Models\Tag;
+use App\Models\Team;
+use App\Support\ValidationPatterns;
+use Illuminate\Console\Command;
+use Illuminate\Support\Facades\Log;
+
+class CleanupNames extends Command
+{
+    protected $signature = 'cleanup:names 
+                            {--dry-run : Preview changes without applying them}
+                            {--model= : Clean specific model (e.g., Project, Server)}
+                            {--backup : Create database backup before changes}
+                            {--force : Skip confirmation prompt}';
+
+    protected $description = 'Sanitize name fields by removing dangerous characters';
+
+    protected array $modelsToClean = [
+        'Project' => Project::class,
+        'Environment' => Environment::class,
+        'Application' => Application::class,
+        'Service' => Service::class,
+        'Server' => Server::class,
+        'Team' => Team::class,
+        'StandalonePostgresql' => StandalonePostgresql::class,
+        'StandaloneMysql' => StandaloneMysql::class,
+        'StandaloneRedis' => StandaloneRedis::class,
+        'StandaloneMongodb' => StandaloneMongodb::class,
+        'StandaloneMariadb' => StandaloneMariadb::class,
+        'StandaloneKeydb' => StandaloneKeydb::class,
+        'StandaloneDragonfly' => StandaloneDragonfly::class,
+        'StandaloneClickhouse' => StandaloneClickhouse::class,
+        'S3Storage' => S3Storage::class,
+        'Tag' => Tag::class,
+        'PrivateKey' => PrivateKey::class,
+        'ScheduledTask' => ScheduledTask::class,
+    ];
+
+    protected array $changes = [];
+
+    protected int $totalProcessed = 0;
+
+    protected int $totalCleaned = 0;
+
+    public function handle(): int
+    {
+        if ($this->option('backup') && ! $this->option('dry-run')) {
+            $this->createBackup();
+        }
+
+        $modelFilter = $this->option('model');
+        $modelsToProcess = $modelFilter
+            ? [$modelFilter => $this->modelsToClean[$modelFilter] ?? null]
+            : $this->modelsToClean;
+
+        if ($modelFilter && ! isset($this->modelsToClean[$modelFilter])) {
+            $this->error("Unknown model: {$modelFilter}");
+            $this->info('Available models: '.implode(', ', array_keys($this->modelsToClean)));
+
+            return self::FAILURE;
+        }
+
+        foreach ($modelsToProcess as $modelName => $modelClass) {
+            if (! $modelClass) {
+                continue;
+            }
+            $this->processModel($modelName, $modelClass);
+        }
+
+        if (! $this->option('dry-run') && $this->totalCleaned > 0) {
+            $this->logChanges();
+        }
+
+        if ($this->option('dry-run')) {
+            $this->info("Name cleanup: would sanitize {$this->totalCleaned} records");
+        } else {
+            $this->info("Name cleanup: sanitized {$this->totalCleaned} records");
+        }
+
+        return self::SUCCESS;
+    }
+
+    protected function processModel(string $modelName, string $modelClass): void
+    {
+        try {
+            $records = $modelClass::all(['id', 'name']);
+            $cleaned = 0;
+
+            foreach ($records as $record) {
+                $this->totalProcessed++;
+
+                $originalName = $record->name;
+                $sanitizedName = $this->sanitizeName($originalName);
+
+                if ($sanitizedName !== $originalName) {
+                    $this->changes[] = [
+                        'model' => $modelName,
+                        'id' => $record->id,
+                        'original' => $originalName,
+                        'sanitized' => $sanitizedName,
+                        'timestamp' => now(),
+                    ];
+
+                    if (! $this->option('dry-run')) {
+                        // Update without triggering events/mutators to avoid conflicts
+                        $modelClass::where('id', $record->id)->update(['name' => $sanitizedName]);
+                    }
+
+                    $cleaned++;
+                    $this->totalCleaned++;
+
+                    // Only log in dry-run mode to preview changes
+                    if ($this->option('dry-run')) {
+                        $this->warn("  🧹 {$modelName} #{$record->id}:");
+                        $this->line('    From: '.$this->truncate($originalName, 80));
+                        $this->line('    To:   '.$this->truncate($sanitizedName, 80));
+                    }
+                }
+            }
+
+        } catch (\Exception $e) {
+            $this->error("Error processing {$modelName}: ".$e->getMessage());
+        }
+    }
+
+    protected function sanitizeName(string $name): string
+    {
+        // Remove all characters that don't match the allowed pattern
+        // Use the shared ValidationPatterns to ensure consistency
+        $allowedPattern = str_replace(['/', '^', '$'], '', ValidationPatterns::NAME_PATTERN);
+        $sanitized = preg_replace('/[^'.$allowedPattern.']+/', '', $name);
+
+        // Clean up excessive whitespace but preserve other allowed characters
+        $sanitized = preg_replace('/\s+/', ' ', $sanitized);
+        $sanitized = trim($sanitized);
+
+        // If result is empty, provide a default name
+        if (empty($sanitized)) {
+            $sanitized = 'sanitized-item';
+        }
+
+        return $sanitized;
+    }
+
+    protected function logChanges(): void
+    {
+        $logFile = storage_path('logs/name-cleanup.log');
+        $logData = [
+            'timestamp' => now()->toISOString(),
+            'total_processed' => $this->totalProcessed,
+            'total_cleaned' => $this->totalCleaned,
+            'changes' => $this->changes,
+        ];
+
+        file_put_contents($logFile, json_encode($logData, JSON_PRETTY_PRINT)."\n", FILE_APPEND);
+
+        Log::info('Name Sanitization completed', [
+            'total_processed' => $this->totalProcessed,
+            'total_sanitized' => $this->totalCleaned,
+            'changes_count' => count($this->changes),
+        ]);
+    }
+
+    protected function createBackup(): void
+    {
+        try {
+            $backupFile = storage_path('backups/name-cleanup-backup-'.now()->format('Y-m-d-H-i-s').'.sql');
+
+            // Ensure backup directory exists
+            if (! file_exists(dirname($backupFile))) {
+                mkdir(dirname($backupFile), 0755, true);
+            }
+
+            $dbConfig = config('database.connections.'.config('database.default'));
+            $command = sprintf(
+                'pg_dump -h %s -p %s -U %s -d %s > %s',
+                $dbConfig['host'],
+                $dbConfig['port'],
+                $dbConfig['username'],
+                $dbConfig['database'],
+                $backupFile
+            );
+
+            exec($command, $output, $returnCode);
+        } catch (\Exception $e) {
+            // Log failure but continue - backup is optional safeguard
+            Log::warning('Name cleanup backup failed', ['error' => $e->getMessage()]);
+        }
+    }
+
+    protected function truncate(string $text, int $length): string
+    {
+        return strlen($text) > $length ? substr($text, 0, $length).'...' : $text;
+    }
+}

app/Console/Commands/CleanupRedis.php

@@ -7,9 +7,9 @@ use Illuminate\Support\Facades\Redis;
 
 class CleanupRedis extends Command
 {
-    protected $signature = 'cleanup:redis {--dry-run : Show what would be deleted without actually deleting} {--skip-overlapping : Skip overlapping queue cleanup}';
+    protected $signature = 'cleanup:redis {--dry-run : Show what would be deleted without actually deleting} {--skip-overlapping : Skip overlapping queue cleanup} {--clear-locks : Clear stale WithoutOverlapping locks} {--restart : Aggressive cleanup mode for system restart (marks all processing jobs as failed)}';
 
-    protected $description = 'Cleanup Redis (Horizon jobs, metrics, overlapping queues, and related data)';
+    protected $description = 'Cleanup Redis (Horizon jobs, metrics, overlapping queues, cache locks, and related data)';
 
     public function handle()
     {
@@ -18,10 +18,6 @@ class CleanupRedis extends Command
         $dryRun = $this->option('dry-run');
         $skipOverlapping = $this->option('skip-overlapping');
 
-        if ($dryRun) {
-            $this->info('DRY RUN MODE - No data will be deleted');
-        }
-
         $deletedCount = 0;
         $totalKeys = 0;
 
@@ -29,8 +25,6 @@ class CleanupRedis extends Command
         $keys = $redis->keys('*');
         $totalKeys = count($keys);
 
-        $this->info("Scanning {$totalKeys} keys for cleanup...");
-
         foreach ($keys as $key) {
             $keyWithoutPrefix = str_replace($prefix, '', $key);
             $type = $redis->command('type', [$keyWithoutPrefix]);
@@ -51,15 +45,27 @@ class CleanupRedis extends Command
 
         // Clean up overlapping queues if not skipped
         if (! $skipOverlapping) {
-            $this->info('Cleaning up overlapping queues...');
             $overlappingCleaned = $this->cleanupOverlappingQueues($redis, $prefix, $dryRun);
             $deletedCount += $overlappingCleaned;
         }
 
+        // Clean up stale cache locks (WithoutOverlapping middleware)
+        if ($this->option('clear-locks')) {
+            $locksCleaned = $this->cleanupCacheLocks($dryRun);
+            $deletedCount += $locksCleaned;
+        }
+
+        // Clean up stuck jobs (restart mode = aggressive, runtime mode = conservative)
+        $isRestart = $this->option('restart');
+        if ($isRestart || $this->option('clear-locks')) {
+            $jobsCleaned = $this->cleanupStuckJobs($redis, $prefix, $dryRun, $isRestart);
+            $deletedCount += $jobsCleaned;
+        }
+
         if ($dryRun) {
-            $this->info("DRY RUN: Would delete {$deletedCount} out of {$totalKeys} keys");
+            $this->info("Redis cleanup: would delete {$deletedCount} items");
         } else {
-            $this->info("Deleted {$deletedCount} out of {$totalKeys} keys");
+            $this->info("Redis cleanup: deleted {$deletedCount} items");
         }
     }
 
@@ -70,11 +76,8 @@ class CleanupRedis extends Command
 
         // Delete completed and failed jobs
         if (in_array($status, ['completed', 'failed'])) {
-            if ($dryRun) {
-                $this->line("Would delete job: {$keyWithoutPrefix} (status: {$status})");
-            } else {
+            if (! $dryRun) {
                 $redis->command('del', [$keyWithoutPrefix]);
-                $this->line("Deleted job: {$keyWithoutPrefix} (status: {$status})");
             }
 
             return true;
@@ -100,11 +103,8 @@ class CleanupRedis extends Command
 
         foreach ($patterns as $pattern => $description) {
             if (str_contains($keyWithoutPrefix, $pattern)) {
-                if ($dryRun) {
-                    $this->line("Would delete {$description}: {$keyWithoutPrefix}");
-                } else {
+                if (! $dryRun) {
                     $redis->command('del', [$keyWithoutPrefix]);
-                    $this->line("Deleted {$description}: {$keyWithoutPrefix}");
                 }
 
                 return true;
@@ -117,11 +117,8 @@ class CleanupRedis extends Command
             $weekAgo = now()->subDays(7)->timestamp;
 
             if ($timestamp < $weekAgo) {
-                if ($dryRun) {
-                    $this->line("Would delete old timestamped data: {$keyWithoutPrefix}");
-                } else {
+                if (! $dryRun) {
                     $redis->command('del', [$keyWithoutPrefix]);
-                    $this->line("Deleted old timestamped data: {$keyWithoutPrefix}");
                 }
 
                 return true;
@@ -145,8 +142,6 @@ class CleanupRedis extends Command
             }
         }
 
-        $this->info('Found '.count($queueKeys).' queue-related keys');
-
         // Group queues by name pattern to find duplicates
         $queueGroups = [];
         foreach ($queueKeys as $queueKey) {
@@ -178,7 +173,6 @@ class CleanupRedis extends Command
     private function deduplicateQueueGroup($redis, $baseName, $keys, $dryRun)
     {
         $cleanedCount = 0;
-        $this->line("Processing queue group: {$baseName} (".count($keys).' keys)');
 
         // Sort keys to keep the most recent one
         usort($keys, function ($a, $b) {
@@ -229,11 +223,8 @@ class CleanupRedis extends Command
             }
 
             if ($shouldDelete) {
-                if ($dryRun) {
-                    $this->line("  Would delete empty queue: {$redundantKey}");
-                } else {
+                if (! $dryRun) {
                     $redis->command('del', [$redundantKey]);
-                    $this->line("  Deleted empty queue: {$redundantKey}");
                 }
                 $cleanedCount++;
             }
@@ -256,15 +247,12 @@ class CleanupRedis extends Command
                 if (count($uniqueItems) < count($items)) {
                     $duplicates = count($items) - count($uniqueItems);
 
-                    if ($dryRun) {
-                        $this->line("  Would remove {$duplicates} duplicate jobs from queue: {$queueKey}");
-                    } else {
+                    if (! $dryRun) {
                         // Rebuild the list with unique items
                         $redis->command('del', [$queueKey]);
                         foreach (array_reverse($uniqueItems) as $item) {
                             $redis->command('lpush', [$queueKey, $item]);
                         }
-                        $this->line("  Removed {$duplicates} duplicate jobs from queue: {$queueKey}");
                     }
                     $cleanedCount += $duplicates;
                 }
@@ -273,4 +261,165 @@ class CleanupRedis extends Command
 
         return $cleanedCount;
     }
+
+    private function cleanupCacheLocks(bool $dryRun): int
+    {
+        $cleanedCount = 0;
+
+        // Use the default Redis connection (database 0) where cache locks are stored
+        $redis = Redis::connection('default');
+
+        // Get all keys matching WithoutOverlapping lock pattern
+        $allKeys = $redis->keys('*');
+        $lockKeys = [];
+
+        foreach ($allKeys as $key) {
+            // Match cache lock keys: they contain 'laravel-queue-overlap'
+            if (preg_match('/overlap/i', $key)) {
+                $lockKeys[] = $key;
+            }
+        }
+        if (empty($lockKeys)) {
+            return 0;
+        }
+
+        foreach ($lockKeys as $lockKey) {
+            // Check TTL to identify stale locks
+            $ttl = $redis->ttl($lockKey);
+
+            // TTL = -1 means no expiration (stale lock!)
+            // TTL = -2 means key doesn't exist
+            // TTL > 0 means lock is valid and will expire
+            if ($ttl === -1) {
+                if ($dryRun) {
+                    $this->warn("  Would delete STALE lock (no expiration): {$lockKey}");
+                } else {
+                    $redis->del($lockKey);
+                }
+                $cleanedCount++;
+            }
+        }
+
+        return $cleanedCount;
+    }
+
+    /**
+     * Clean up stuck jobs based on mode (restart vs runtime).
+     *
+     * @param  mixed  $redis  Redis connection
+     * @param  string  $prefix  Horizon prefix
+     * @param  bool  $dryRun  Dry run mode
+     * @param  bool  $isRestart  Restart mode (aggressive) vs runtime mode (conservative)
+     * @return int Number of jobs cleaned
+     */
+    private function cleanupStuckJobs($redis, string $prefix, bool $dryRun, bool $isRestart): int
+    {
+        $cleanedCount = 0;
+        $now = time();
+
+        // Get all keys with the horizon prefix
+        $cursor = 0;
+        $keys = [];
+        do {
+            $result = $redis->scan($cursor, ['match' => '*', 'count' => 100]);
+
+            // Guard against scan() returning false
+            if ($result === false) {
+                $this->error('Redis scan failed, stopping key retrieval');
+                break;
+            }
+
+            $cursor = $result[0];
+            $keys = array_merge($keys, $result[1]);
+        } while ($cursor !== 0);
+
+        foreach ($keys as $key) {
+            $keyWithoutPrefix = str_replace($prefix, '', $key);
+            $type = $redis->command('type', [$keyWithoutPrefix]);
+
+            // Only process hash-type keys (individual jobs)
+            if ($type !== 5) {
+                continue;
+            }
+
+            $data = $redis->command('hgetall', [$keyWithoutPrefix]);
+            $status = data_get($data, 'status');
+            $payload = data_get($data, 'payload');
+
+            // Only process jobs in "processing" or "reserved" state
+            if (! in_array($status, ['processing', 'reserved'])) {
+                continue;
+            }
+
+            // Parse job payload to get job class and started time
+            $payloadData = json_decode($payload, true);
+
+            // Check for JSON decode errors
+            if ($payloadData === null || json_last_error() !== JSON_ERROR_NONE) {
+                $errorMsg = json_last_error_msg();
+                $truncatedPayload = is_string($payload) ? substr($payload, 0, 200) : 'non-string payload';
+                $this->error("Failed to decode job payload for {$keyWithoutPrefix}: {$errorMsg}. Payload: {$truncatedPayload}");
+
+                continue;
+            }
+
+            $jobClass = data_get($payloadData, 'displayName', 'Unknown');
+
+            // Prefer reserved_at (when job started processing), fallback to created_at
+            $reservedAt = (int) data_get($data, 'reserved_at', 0);
+            $createdAt = (int) data_get($data, 'created_at', 0);
+            $startTime = $reservedAt ?: $createdAt;
+
+            // If we can't determine when the job started, skip it
+            if (! $startTime) {
+                continue;
+            }
+
+            // Calculate how long the job has been processing
+            $processingTime = $now - $startTime;
+
+            $shouldFail = false;
+            $reason = '';
+
+            if ($isRestart) {
+                // RESTART MODE: Mark ALL processing/reserved jobs as failed
+                // Safe because all workers are dead on restart
+                $shouldFail = true;
+                $reason = 'System restart - all workers terminated';
+            } else {
+                // RUNTIME MODE: Only mark truly stuck jobs as failed
+                // Be conservative to avoid killing legitimate long-running jobs
+
+                // Skip ApplicationDeploymentJob entirely (has dynamic_timeout, can run 2+ hours)
+                if (str_contains($jobClass, 'ApplicationDeploymentJob')) {
+                    continue;
+                }
+
+                // Skip DatabaseBackupJob (large backups can take hours)
+                if (str_contains($jobClass, 'DatabaseBackupJob')) {
+                    continue;
+                }
+
+                // For other jobs, only fail if processing > 12 hours
+                if ($processingTime > 43200) { // 12 hours
+                    $shouldFail = true;
+                    $reason = 'Processing for more than 12 hours';
+                }
+            }
+
+            if ($shouldFail) {
+                if ($dryRun) {
+                    $this->warn("  Would mark as FAILED: {$jobClass} (processing for ".round($processingTime / 60, 1)." min) - {$reason}");
+                } else {
+                    // Mark job as failed
+                    $redis->command('hset', [$keyWithoutPrefix, 'status', 'failed']);
+                    $redis->command('hset', [$keyWithoutPrefix, 'failed_at', $now]);
+                    $redis->command('hset', [$keyWithoutPrefix, 'exception', "Job cleaned up by cleanup:redis - {$reason}"]);
+                }
+                $cleanedCount++;
+            }
+        }
+
+        return $cleanedCount;
+    }
 }

app/Console/Commands/CleanupStuckedResources.php

@@ -3,6 +3,7 @@
 namespace App\Console\Commands;
 
 use App\Jobs\CleanupHelperContainersJob;
+use App\Jobs\DeleteResourceJob;
 use App\Models\Application;
 use App\Models\ApplicationDeploymentQueue;
 use App\Models\ApplicationPreview;
@@ -12,6 +13,7 @@ use App\Models\Server;
 use App\Models\Service;
 use App\Models\ServiceApplication;
 use App\Models\ServiceDatabase;
+use App\Models\SslCertificate;
 use App\Models\StandaloneClickhouse;
 use App\Models\StandaloneDragonfly;
 use App\Models\StandaloneKeydb;
@@ -57,6 +59,15 @@ class CleanupStuckedResources extends Command
         } catch (\Throwable $e) {
             echo "Error in cleaning stucked resources: {$e->getMessage()}\n";
         }
+        try {
+            $servers = Server::onlyTrashed()->get();
+            foreach ($servers as $server) {
+                echo "Force deleting stuck server: {$server->name}\n";
+                $server->forceDelete();
+            }
+        } catch (\Throwable $e) {
+            echo "Error in cleaning stuck servers: {$e->getMessage()}\n";
+        }
         try {
             $applicationsDeploymentQueue = ApplicationDeploymentQueue::get();
             foreach ($applicationsDeploymentQueue as $applicationDeploymentQueue) {
@@ -72,7 +83,7 @@ class CleanupStuckedResources extends Command
             $applications = Application::withTrashed()->whereNotNull('deleted_at')->get();
             foreach ($applications as $application) {
                 echo "Deleting stuck application: {$application->name}\n";
-                $application->forceDelete();
+                DeleteResourceJob::dispatch($application);
             }
         } catch (\Throwable $e) {
             echo "Error in cleaning stuck application: {$e->getMessage()}\n";
@@ -82,26 +93,35 @@ class CleanupStuckedResources extends Command
             foreach ($applicationsPreviews as $applicationPreview) {
                 if (! data_get($applicationPreview, 'application')) {
                     echo "Deleting stuck application preview: {$applicationPreview->uuid}\n";
-                    $applicationPreview->delete();
+                    DeleteResourceJob::dispatch($applicationPreview);
                 }
             }
         } catch (\Throwable $e) {
             echo "Error in cleaning stuck application: {$e->getMessage()}\n";
         }
+        try {
+            $applicationsPreviews = ApplicationPreview::withTrashed()->whereNotNull('deleted_at')->get();
+            foreach ($applicationsPreviews as $applicationPreview) {
+                echo "Deleting stuck application preview: {$applicationPreview->fqdn}\n";
+                DeleteResourceJob::dispatch($applicationPreview);
+            }
+        } catch (\Throwable $e) {
+            echo "Error in cleaning stuck application: {$e->getMessage()}\n";
+        }
         try {
             $postgresqls = StandalonePostgresql::withTrashed()->whereNotNull('deleted_at')->get();
             foreach ($postgresqls as $postgresql) {
                 echo "Deleting stuck postgresql: {$postgresql->name}\n";
-                $postgresql->forceDelete();
+                DeleteResourceJob::dispatch($postgresql);
             }
         } catch (\Throwable $e) {
             echo "Error in cleaning stuck postgresql: {$e->getMessage()}\n";
         }
         try {
-            $redis = StandaloneRedis::withTrashed()->whereNotNull('deleted_at')->get();
-            foreach ($redis as $redis) {
+            $rediss = StandaloneRedis::withTrashed()->whereNotNull('deleted_at')->get();
+            foreach ($rediss as $redis) {
                 echo "Deleting stuck redis: {$redis->name}\n";
-                $redis->forceDelete();
+                DeleteResourceJob::dispatch($redis);
             }
         } catch (\Throwable $e) {
             echo "Error in cleaning stuck redis: {$e->getMessage()}\n";
@@ -110,7 +130,7 @@ class CleanupStuckedResources extends Command
             $keydbs = StandaloneKeydb::withTrashed()->whereNotNull('deleted_at')->get();
             foreach ($keydbs as $keydb) {
                 echo "Deleting stuck keydb: {$keydb->name}\n";
-                $keydb->forceDelete();
+                DeleteResourceJob::dispatch($keydb);
             }
         } catch (\Throwable $e) {
             echo "Error in cleaning stuck keydb: {$e->getMessage()}\n";
@@ -119,7 +139,7 @@ class CleanupStuckedResources extends Command
             $dragonflies = StandaloneDragonfly::withTrashed()->whereNotNull('deleted_at')->get();
             foreach ($dragonflies as $dragonfly) {
                 echo "Deleting stuck dragonfly: {$dragonfly->name}\n";
-                $dragonfly->forceDelete();
+                DeleteResourceJob::dispatch($dragonfly);
             }
         } catch (\Throwable $e) {
             echo "Error in cleaning stuck dragonfly: {$e->getMessage()}\n";
@@ -128,7 +148,7 @@ class CleanupStuckedResources extends Command
             $clickhouses = StandaloneClickhouse::withTrashed()->whereNotNull('deleted_at')->get();
             foreach ($clickhouses as $clickhouse) {
                 echo "Deleting stuck clickhouse: {$clickhouse->name}\n";
-                $clickhouse->forceDelete();
+                DeleteResourceJob::dispatch($clickhouse);
             }
         } catch (\Throwable $e) {
             echo "Error in cleaning stuck clickhouse: {$e->getMessage()}\n";
@@ -137,7 +157,7 @@ class CleanupStuckedResources extends Command
             $mongodbs = StandaloneMongodb::withTrashed()->whereNotNull('deleted_at')->get();
             foreach ($mongodbs as $mongodb) {
                 echo "Deleting stuck mongodb: {$mongodb->name}\n";
-                $mongodb->forceDelete();
+                DeleteResourceJob::dispatch($mongodb);
             }
         } catch (\Throwable $e) {
             echo "Error in cleaning stuck mongodb: {$e->getMessage()}\n";
@@ -146,7 +166,7 @@ class CleanupStuckedResources extends Command
             $mysqls = StandaloneMysql::withTrashed()->whereNotNull('deleted_at')->get();
             foreach ($mysqls as $mysql) {
                 echo "Deleting stuck mysql: {$mysql->name}\n";
-                $mysql->forceDelete();
+                DeleteResourceJob::dispatch($mysql);
             }
         } catch (\Throwable $e) {
             echo "Error in cleaning stuck mysql: {$e->getMessage()}\n";
@@ -155,7 +175,7 @@ class CleanupStuckedResources extends Command
             $mariadbs = StandaloneMariadb::withTrashed()->whereNotNull('deleted_at')->get();
             foreach ($mariadbs as $mariadb) {
                 echo "Deleting stuck mariadb: {$mariadb->name}\n";
-                $mariadb->forceDelete();
+                DeleteResourceJob::dispatch($mariadb);
             }
         } catch (\Throwable $e) {
             echo "Error in cleaning stuck mariadb: {$e->getMessage()}\n";
@@ -164,7 +184,7 @@ class CleanupStuckedResources extends Command
             $services = Service::withTrashed()->whereNotNull('deleted_at')->get();
             foreach ($services as $service) {
                 echo "Deleting stuck service: {$service->name}\n";
-                $service->forceDelete();
+                DeleteResourceJob::dispatch($service);
             }
         } catch (\Throwable $e) {
             echo "Error in cleaning stuck service: {$e->getMessage()}\n";
@@ -202,9 +222,14 @@ class CleanupStuckedResources extends Command
         try {
             $scheduled_backups = ScheduledDatabaseBackup::all();
             foreach ($scheduled_backups as $scheduled_backup) {
-                if (! $scheduled_backup->server()) {
-                    echo "Deleting stuck scheduledbackup: {$scheduled_backup->name}\n";
-                    $scheduled_backup->delete();
+                try {
+                    $server = $scheduled_backup->server();
+                    if (! $server) {
+                        echo "Deleting stuck scheduledbackup: {$scheduled_backup->name}\n";
+                        $scheduled_backup->delete();
+                    }
+                } catch (\Throwable $e) {
+                    echo "Error checking server for scheduledbackup {$scheduled_backup->id}: {$e->getMessage()}\n";
                 }
             }
         } catch (\Throwable $e) {
@@ -217,19 +242,19 @@ class CleanupStuckedResources extends Command
             foreach ($applications as $application) {
                 if (! data_get($application, 'environment')) {
                     echo 'Application without environment: '.$application->name.'\n';
-                    $application->forceDelete();
+                    DeleteResourceJob::dispatch($application);
 
                     continue;
                 }
                 if (! $application->destination()) {
                     echo 'Application without destination: '.$application->name.'\n';
-                    $application->forceDelete();
+                    DeleteResourceJob::dispatch($application);
 
                     continue;
                 }
                 if (! data_get($application, 'destination.server')) {
                     echo 'Application without server: '.$application->name.'\n';
-                    $application->forceDelete();
+                    DeleteResourceJob::dispatch($application);
 
                     continue;
                 }
@@ -242,19 +267,19 @@ class CleanupStuckedResources extends Command
             foreach ($postgresqls as $postgresql) {
                 if (! data_get($postgresql, 'environment')) {
                     echo 'Postgresql without environment: '.$postgresql->name.'\n';
-                    $postgresql->forceDelete();
+                    DeleteResourceJob::dispatch($postgresql);
 
                     continue;
                 }
                 if (! $postgresql->destination()) {
                     echo 'Postgresql without destination: '.$postgresql->name.'\n';
-                    $postgresql->forceDelete();
+                    DeleteResourceJob::dispatch($postgresql);
 
                     continue;
                 }
                 if (! data_get($postgresql, 'destination.server')) {
                     echo 'Postgresql without server: '.$postgresql->name.'\n';
-                    $postgresql->forceDelete();
+                    DeleteResourceJob::dispatch($postgresql);
 
                     continue;
                 }
@@ -267,19 +292,19 @@ class CleanupStuckedResources extends Command
             foreach ($redis as $redis) {
                 if (! data_get($redis, 'environment')) {
                     echo 'Redis without environment: '.$redis->name.'\n';
-                    $redis->forceDelete();
+                    DeleteResourceJob::dispatch($redis);
 
                     continue;
                 }
                 if (! $redis->destination()) {
                     echo 'Redis without destination: '.$redis->name.'\n';
-                    $redis->forceDelete();
+                    DeleteResourceJob::dispatch($redis);
 
                     continue;
                 }
                 if (! data_get($redis, 'destination.server')) {
                     echo 'Redis without server: '.$redis->name.'\n';
-                    $redis->forceDelete();
+                    DeleteResourceJob::dispatch($redis);
 
                     continue;
                 }
@@ -293,19 +318,19 @@ class CleanupStuckedResources extends Command
             foreach ($mongodbs as $mongodb) {
                 if (! data_get($mongodb, 'environment')) {
                     echo 'Mongodb without environment: '.$mongodb->name.'\n';
-                    $mongodb->forceDelete();
+                    DeleteResourceJob::dispatch($mongodb);
 
                     continue;
                 }
                 if (! $mongodb->destination()) {
                     echo 'Mongodb without destination: '.$mongodb->name.'\n';
-                    $mongodb->forceDelete();
+                    DeleteResourceJob::dispatch($mongodb);
 
                     continue;
                 }
                 if (! data_get($mongodb, 'destination.server')) {
                     echo 'Mongodb without server:  '.$mongodb->name.'\n';
-                    $mongodb->forceDelete();
+                    DeleteResourceJob::dispatch($mongodb);
 
                     continue;
                 }
@@ -319,19 +344,19 @@ class CleanupStuckedResources extends Command
             foreach ($mysqls as $mysql) {
                 if (! data_get($mysql, 'environment')) {
                     echo 'Mysql without environment: '.$mysql->name.'\n';
-                    $mysql->forceDelete();
+                    DeleteResourceJob::dispatch($mysql);
 
                     continue;
                 }
                 if (! $mysql->destination()) {
                     echo 'Mysql without destination: '.$mysql->name.'\n';
-                    $mysql->forceDelete();
+                    DeleteResourceJob::dispatch($mysql);
 
                     continue;
                 }
                 if (! data_get($mysql, 'destination.server')) {
                     echo 'Mysql without server: '.$mysql->name.'\n';
-                    $mysql->forceDelete();
+                    DeleteResourceJob::dispatch($mysql);
 
                     continue;
                 }
@@ -345,19 +370,19 @@ class CleanupStuckedResources extends Command
             foreach ($mariadbs as $mariadb) {
                 if (! data_get($mariadb, 'environment')) {
                     echo 'Mariadb without environment: '.$mariadb->name.'\n';
-                    $mariadb->forceDelete();
+                    DeleteResourceJob::dispatch($mariadb);
 
                     continue;
                 }
                 if (! $mariadb->destination()) {
                     echo 'Mariadb without destination: '.$mariadb->name.'\n';
-                    $mariadb->forceDelete();
+                    DeleteResourceJob::dispatch($mariadb);
 
                     continue;
                 }
                 if (! data_get($mariadb, 'destination.server')) {
                     echo 'Mariadb without server: '.$mariadb->name.'\n';
-                    $mariadb->forceDelete();
+                    DeleteResourceJob::dispatch($mariadb);
 
                     continue;
                 }
@@ -371,19 +396,19 @@ class CleanupStuckedResources extends Command
             foreach ($services as $service) {
                 if (! data_get($service, 'environment')) {
                     echo 'Service without environment: '.$service->name.'\n';
-                    $service->forceDelete();
+                    DeleteResourceJob::dispatch($service);
 
                     continue;
                 }
                 if (! $service->destination()) {
                     echo 'Service without destination: '.$service->name.'\n';
-                    $service->forceDelete();
+                    DeleteResourceJob::dispatch($service);
 
                     continue;
                 }
                 if (! data_get($service, 'server')) {
                     echo 'Service without server: '.$service->name.'\n';
-                    $service->forceDelete();
+                    DeleteResourceJob::dispatch($service);
 
                     continue;
                 }
@@ -417,5 +442,18 @@ class CleanupStuckedResources extends Command
         } catch (\Throwable $e) {
             echo "Error in ServiceDatabases: {$e->getMessage()}\n";
         }
+
+        try {
+            $orphanedCerts = SslCertificate::whereNotIn('server_id', function ($query) {
+                $query->select('id')->from('servers');
+            })->get();
+
+            foreach ($orphanedCerts as $cert) {
+                echo "Deleting orphaned SSL certificate: {$cert->id} (server_id: {$cert->server_id})\n";
+                $cert->delete();
+            }
+        } catch (\Throwable $e) {
+            echo "Error in cleaning orphaned SSL certificates: {$e->getMessage()}\n";
+        }
     }
 }

app/Console/Commands/CleanupUnreachableServers.php

@@ -14,13 +14,17 @@ class CleanupUnreachableServers extends Command
     public function handle()
     {
         echo "Running unreachable server cleanup...\n";
-        $servers = Server::where('unreachable_count', 3)->where('unreachable_notification_sent', true)->where('updated_at', '<', now()->subDays(7))->get();
+        $servers = Server::where('unreachable_count', '>=', 3)->where('unreachable_notification_sent', true)->where('updated_at', '<', now()->subDays(7))->get();
         if ($servers->count() > 0) {
             foreach ($servers as $server) {
                 echo "Cleanup unreachable server ($server->id) with name $server->name";
-                $server->update([
-                    'ip' => '1.2.3.4',
-                ]);
+                if (isCloud()) {
+                    $server->update([
+                        'ip' => '1.2.3.4',
+                    ]);
+                } else {
+                    $server->forceDisableServer();
+                }
             }
         }
     }

app/Console/Commands/ClearGlobalSearchCache.php

@@ -0,0 +1,83 @@
+<?php
+
+namespace App\Console\Commands;
+
+use App\Livewire\GlobalSearch;
+use App\Models\Team;
+use Illuminate\Console\Command;
+use Illuminate\Support\Facades\Cache;
+
+class ClearGlobalSearchCache extends Command
+{
+    /**
+     * The name and signature of the console command.
+     */
+    protected $signature = 'search:clear {--team= : Clear cache for specific team ID} {--all : Clear cache for all teams}';
+
+    /**
+     * The console command description.
+     */
+    protected $description = 'Clear the global search cache for testing or manual refresh';
+
+    /**
+     * Execute the console command.
+     */
+    public function handle(): int
+    {
+        if ($this->option('all')) {
+            return $this->clearAllTeamsCache();
+        }
+
+        if ($teamId = $this->option('team')) {
+            return $this->clearTeamCache($teamId);
+        }
+
+        // If no options provided, clear cache for current user's team
+        if (! auth()->check()) {
+            $this->error('No authenticated user found. Use --team=ID or --all option.');
+
+            return Command::FAILURE;
+        }
+
+        $teamId = auth()->user()->currentTeam()->id;
+
+        return $this->clearTeamCache($teamId);
+    }
+
+    private function clearTeamCache(int $teamId): int
+    {
+        $team = Team::find($teamId);
+
+        if (! $team) {
+            $this->error("Team with ID {$teamId} not found.");
+
+            return Command::FAILURE;
+        }
+
+        GlobalSearch::clearTeamCache($teamId);
+        $this->info("✓ Cleared global search cache for team: {$team->name} (ID: {$teamId})");
+
+        return Command::SUCCESS;
+    }
+
+    private function clearAllTeamsCache(): int
+    {
+        $teams = Team::all();
+
+        if ($teams->isEmpty()) {
+            $this->warn('No teams found.');
+
+            return Command::SUCCESS;
+        }
+
+        $count = 0;
+        foreach ($teams as $team) {
+            GlobalSearch::clearTeamCache($team->id);
+            $count++;
+        }
+
+        $this->info("✓ Cleared global search cache for {$count} team(s)");
+
+        return Command::SUCCESS;
+    }
+}

app/Console/Commands/Cloud/CloudFixSubscription.php

@@ -0,0 +1,881 @@
+<?php
+
+namespace App\Console\Commands\Cloud;
+
+use App\Models\Team;
+use Illuminate\Console\Command;
+use Stripe\Exception\InvalidRequestException;
+use Stripe\StripeClient;
+
+class CloudFixSubscription extends Command
+{
+    /**
+     * The name and signature of the console command.
+     *
+     * @var string
+     */
+    protected $signature = 'cloud:fix-subscription
+                            {--fix-canceled-subs : Fix canceled subscriptions in database}
+                            {--verify-all : Verify all active subscriptions against Stripe}
+                            {--fix-verified : Fix discrepancies found during verification}
+                            {--dry-run : Show what would be fixed without making changes}
+                            {--one : Only fix the first found subscription}';
+
+    /**
+     * The console command description.
+     *
+     * @var string
+     */
+    protected $description = 'Fix Cloud subscriptions';
+
+    /**
+     * Execute the console command.
+     */
+    public function handle()
+    {
+        $stripe = app(StripeClient::class);
+
+        if ($this->option('verify-all')) {
+            return $this->verifyAllActiveSubscriptions($stripe);
+        }
+
+        if ($this->option('fix-canceled-subs') || $this->option('dry-run')) {
+            return $this->fixCanceledSubscriptions($stripe);
+        }
+
+        $activeSubscribers = Team::whereRelation('subscription', 'stripe_invoice_paid', true)->get();
+
+        $out = fopen('php://output', 'w');
+        // CSV header
+        fputcsv($out, [
+            'team_id',
+            'invoice_status',
+            'stripe_customer_url',
+            'stripe_subscription_id',
+            'subscription_status',
+            'subscription_url',
+            'note',
+        ]);
+
+        foreach ($activeSubscribers as $team) {
+            $stripeSubscriptionId = $team->subscription->stripe_subscription_id;
+            $stripeInvoicePaid = $team->subscription->stripe_invoice_paid;
+            $stripeCustomerId = $team->subscription->stripe_customer_id;
+
+            if (! $stripeSubscriptionId && str($stripeInvoicePaid)->lower() != 'past_due') {
+                fputcsv($out, [
+                    $team->id,
+                    $stripeInvoicePaid,
+                    $stripeCustomerId ? "https://dashboard.stripe.com/customers/{$stripeCustomerId}" : null,
+                    null,
+                    null,
+                    null,
+                    'Missing subscription ID while invoice not past_due',
+                ]);
+
+                continue;
+            }
+
+            if (! $stripeSubscriptionId) {
+                // No subscription ID and invoice is past_due, still record for visibility
+                fputcsv($out, [
+                    $team->id,
+                    $stripeInvoicePaid,
+                    $stripeCustomerId ? "https://dashboard.stripe.com/customers/{$stripeCustomerId}" : null,
+                    null,
+                    null,
+                    null,
+                    'Missing subscription ID',
+                ]);
+
+                continue;
+            }
+
+            $subscription = $stripe->subscriptions->retrieve($stripeSubscriptionId);
+            if ($subscription->status === 'active') {
+                continue;
+            }
+
+            fputcsv($out, [
+                $team->id,
+                $stripeInvoicePaid,
+                $stripeCustomerId ? "https://dashboard.stripe.com/customers/{$stripeCustomerId}" : null,
+                $stripeSubscriptionId,
+                $subscription->status,
+                "https://dashboard.stripe.com/subscriptions/{$stripeSubscriptionId}",
+                'Subscription not active',
+            ]);
+        }
+
+        fclose($out);
+    }
+
+    /**
+     * Fix canceled subscriptions in the database
+     */
+    private function fixCanceledSubscriptions(StripeClient $stripe)
+    {
+        $isDryRun = $this->option('dry-run');
+        $checkOne = $this->option('one');
+
+        if ($isDryRun) {
+            $this->info('DRY RUN MODE - No changes will be made');
+            if ($checkOne) {
+                $this->info('Checking only the first canceled subscription...');
+            } else {
+                $this->info('Checking for canceled subscriptions...');
+            }
+        } else {
+            if ($checkOne) {
+                $this->info('Checking and fixing only the first canceled subscription...');
+            } else {
+                $this->info('Checking and fixing canceled subscriptions...');
+            }
+        }
+
+        $teamsWithSubscriptions = Team::whereRelation('subscription', 'stripe_invoice_paid', true)->get();
+        $toFixCount = 0;
+        $fixedCount = 0;
+        $errors = [];
+        $canceledSubscriptions = [];
+
+        foreach ($teamsWithSubscriptions as $team) {
+            $subscription = $team->subscription;
+
+            if (! $subscription->stripe_subscription_id) {
+                continue;
+            }
+
+            try {
+                $stripeSubscription = $stripe->subscriptions->retrieve(
+                    $subscription->stripe_subscription_id
+                );
+
+                if ($stripeSubscription->status === 'canceled') {
+                    $toFixCount++;
+
+                    // Get team members' emails
+                    $memberEmails = $team->members->pluck('email')->toArray();
+
+                    $canceledSubscriptions[] = [
+                        'team_id' => $team->id,
+                        'team_name' => $team->name,
+                        'customer_id' => $subscription->stripe_customer_id,
+                        'subscription_id' => $subscription->stripe_subscription_id,
+                        'status' => 'canceled',
+                        'member_emails' => $memberEmails,
+                        'subscription_model' => $subscription->toArray(),
+                    ];
+
+                    if ($isDryRun) {
+                        $this->warn('Would fix canceled subscription:');
+                        $this->line("  Team ID: {$team->id}");
+                        $this->line("  Team Name: {$team->name}");
+                        $this->line('  Team Members: '.implode(', ', $memberEmails));
+                        $this->line("  Customer URL: https://dashboard.stripe.com/customers/{$subscription->stripe_customer_id}");
+                        $this->line("  Subscription URL: https://dashboard.stripe.com/subscriptions/{$subscription->stripe_subscription_id}");
+                        $this->line('  Current Subscription Data:');
+                        foreach ($subscription->getAttributes() as $key => $value) {
+                            if (is_null($value)) {
+                                $this->line("    - {$key}: null");
+                            } elseif (is_bool($value)) {
+                                $this->line("    - {$key}: ".($value ? 'true' : 'false'));
+                            } else {
+                                $this->line("    - {$key}: {$value}");
+                            }
+                        }
+                        $this->newLine();
+                    } else {
+                        $this->warn("Found canceled subscription for Team ID: {$team->id}");
+
+                        // Send internal notification with all details before fixing
+                        $notificationMessage = "Fixing canceled subscription:\n";
+                        $notificationMessage .= "Team ID: {$team->id}\n";
+                        $notificationMessage .= "Team Name: {$team->name}\n";
+                        $notificationMessage .= 'Team Members: '.implode(', ', $memberEmails)."\n";
+                        $notificationMessage .= "Customer URL: https://dashboard.stripe.com/customers/{$subscription->stripe_customer_id}\n";
+                        $notificationMessage .= "Subscription URL: https://dashboard.stripe.com/subscriptions/{$subscription->stripe_subscription_id}\n";
+                        $notificationMessage .= "Subscription Data:\n";
+                        foreach ($subscription->getAttributes() as $key => $value) {
+                            if (is_null($value)) {
+                                $notificationMessage .= "  - {$key}: null\n";
+                            } elseif (is_bool($value)) {
+                                $notificationMessage .= "  - {$key}: ".($value ? 'true' : 'false')."\n";
+                            } else {
+                                $notificationMessage .= "  - {$key}: {$value}\n";
+                            }
+                        }
+                        send_internal_notification($notificationMessage);
+
+                        // Apply the same logic as customer.subscription.deleted webhook
+                        $team->subscriptionEnded();
+
+                        $fixedCount++;
+                        $this->info("  ✓ Fixed subscription for Team ID: {$team->id}");
+                        $this->line('    Team Members: '.implode(', ', $memberEmails));
+                        $this->line("    Customer URL: https://dashboard.stripe.com/customers/{$subscription->stripe_customer_id}");
+                        $this->line("    Subscription URL: https://dashboard.stripe.com/subscriptions/{$subscription->stripe_subscription_id}");
+                    }
+
+                    // Break if --one flag is set
+                    if ($checkOne) {
+                        break;
+                    }
+                }
+            } catch (InvalidRequestException $e) {
+                if ($e->getStripeCode() === 'resource_missing') {
+                    $toFixCount++;
+
+                    // Get team members' emails
+                    $memberEmails = $team->members->pluck('email')->toArray();
+
+                    $canceledSubscriptions[] = [
+                        'team_id' => $team->id,
+                        'team_name' => $team->name,
+                        'customer_id' => $subscription->stripe_customer_id,
+                        'subscription_id' => $subscription->stripe_subscription_id,
+                        'status' => 'missing',
+                        'member_emails' => $memberEmails,
+                        'subscription_model' => $subscription->toArray(),
+                    ];
+
+                    if ($isDryRun) {
+                        $this->error('Would fix missing subscription (not found in Stripe):');
+                        $this->line("  Team ID: {$team->id}");
+                        $this->line("  Team Name: {$team->name}");
+                        $this->line('  Team Members: '.implode(', ', $memberEmails));
+                        $this->line("  Customer URL: https://dashboard.stripe.com/customers/{$subscription->stripe_customer_id}");
+                        $this->line("  Subscription ID (missing): {$subscription->stripe_subscription_id}");
+                        $this->line('  Current Subscription Data:');
+                        foreach ($subscription->getAttributes() as $key => $value) {
+                            if (is_null($value)) {
+                                $this->line("    - {$key}: null");
+                            } elseif (is_bool($value)) {
+                                $this->line("    - {$key}: ".($value ? 'true' : 'false'));
+                            } else {
+                                $this->line("    - {$key}: {$value}");
+                            }
+                        }
+                        $this->newLine();
+                    } else {
+                        $this->error("Subscription not found in Stripe for Team ID: {$team->id}");
+
+                        // Send internal notification with all details before fixing
+                        $notificationMessage = "Fixing missing subscription (not found in Stripe):\n";
+                        $notificationMessage .= "Team ID: {$team->id}\n";
+                        $notificationMessage .= "Team Name: {$team->name}\n";
+                        $notificationMessage .= 'Team Members: '.implode(', ', $memberEmails)."\n";
+                        $notificationMessage .= "Customer URL: https://dashboard.stripe.com/customers/{$subscription->stripe_customer_id}\n";
+                        $notificationMessage .= "Subscription ID (missing): {$subscription->stripe_subscription_id}\n";
+                        $notificationMessage .= "Subscription Data:\n";
+                        foreach ($subscription->getAttributes() as $key => $value) {
+                            if (is_null($value)) {
+                                $notificationMessage .= "  - {$key}: null\n";
+                            } elseif (is_bool($value)) {
+                                $notificationMessage .= "  - {$key}: ".($value ? 'true' : 'false')."\n";
+                            } else {
+                                $notificationMessage .= "  - {$key}: {$value}\n";
+                            }
+                        }
+                        send_internal_notification($notificationMessage);
+
+                        // Apply the same logic as customer.subscription.deleted webhook
+                        $team->subscriptionEnded();
+
+                        $fixedCount++;
+                        $this->info("  ✓ Fixed missing subscription for Team ID: {$team->id}");
+                        $this->line('    Team Members: '.implode(', ', $memberEmails));
+                        $this->line("    Customer URL: https://dashboard.stripe.com/customers/{$subscription->stripe_customer_id}");
+                    }
+
+                    // Break if --one flag is set
+                    if ($checkOne) {
+                        break;
+                    }
+                } else {
+                    $errors[] = "Team ID {$team->id}: ".$e->getMessage();
+                }
+            } catch (\Exception $e) {
+                $errors[] = "Team ID {$team->id}: ".$e->getMessage();
+            }
+        }
+
+        $this->newLine();
+        $this->info('Summary:');
+
+        if ($isDryRun) {
+            $this->info("  - Found {$toFixCount} canceled/missing subscriptions that would be fixed");
+
+            if ($toFixCount > 0) {
+                $this->newLine();
+                $this->comment('Run with --fix-canceled-subs to apply these changes');
+            }
+        } else {
+            $this->info("  - Fixed {$fixedCount} canceled/missing subscriptions");
+        }
+
+        if (! empty($errors)) {
+            $this->newLine();
+            $this->error('Errors encountered:');
+            foreach ($errors as $error) {
+                $this->error("  - {$error}");
+            }
+        }
+
+        return 0;
+    }
+
+    /**
+     * Verify all active subscriptions against Stripe API
+     */
+    private function verifyAllActiveSubscriptions(StripeClient $stripe)
+    {
+        $isDryRun = $this->option('dry-run');
+        $shouldFix = $this->option('fix-verified');
+
+        $this->info('Verifying all active subscriptions against Stripe...');
+        if ($isDryRun) {
+            $this->info('DRY RUN MODE - No changes will be made');
+        }
+        if ($shouldFix && ! $isDryRun) {
+            $this->warn('FIX MODE - Discrepancies will be corrected');
+        }
+
+        // Get all teams with active subscriptions
+        $teamsWithActiveSubscriptions = Team::whereRelation('subscription', 'stripe_invoice_paid', true)->get();
+        $totalCount = $teamsWithActiveSubscriptions->count();
+
+        $this->info("Found {$totalCount} teams with active subscriptions in database");
+        $this->newLine();
+
+        $out = fopen('php://output', 'w');
+
+        // CSV header
+        fputcsv($out, [
+            'team_id',
+            'team_name',
+            'customer_id',
+            'subscription_id',
+            'db_status',
+            'stripe_status',
+            'action',
+            'member_emails',
+            'customer_url',
+            'subscription_url',
+        ]);
+
+        $stats = [
+            'total' => $totalCount,
+            'valid_active' => 0,
+            'valid_past_due' => 0,
+            'canceled' => 0,
+            'missing' => 0,
+            'invalid' => 0,
+            'fixed' => 0,
+            'errors' => 0,
+        ];
+
+        $processedCount = 0;
+
+        foreach ($teamsWithActiveSubscriptions as $team) {
+            $subscription = $team->subscription;
+            $memberEmails = $team->members->pluck('email')->toArray();
+
+            // Database state
+            $dbStatus = 'active';
+            if ($subscription->stripe_past_due) {
+                $dbStatus = 'past_due';
+            }
+
+            $stripeStatus = null;
+            $action = 'none';
+
+            if (! $subscription->stripe_subscription_id) {
+                $this->line("Team {$team->id}: Missing subscription ID, searching in Stripe...");
+
+                $foundResult = null;
+                $searchMethod = null;
+
+                // Search by customer ID
+                if ($subscription->stripe_customer_id) {
+                    $this->line("  → Searching by customer ID: {$subscription->stripe_customer_id}");
+                    $foundResult = $this->searchSubscriptionsByCustomer($stripe, $subscription->stripe_customer_id);
+                    if ($foundResult) {
+                        $searchMethod = $foundResult['method'];
+                    }
+                } else {
+                    $this->line('  → No customer ID available');
+                }
+
+                // Search by emails if not found
+                if (! $foundResult && count($memberEmails) > 0) {
+                    $foundResult = $this->searchSubscriptionsByEmails($stripe, $memberEmails);
+                    if ($foundResult) {
+                        $searchMethod = $foundResult['method'];
+
+                        // Update customer ID if different
+                        if (isset($foundResult['customer_id']) && $subscription->stripe_customer_id !== $foundResult['customer_id']) {
+                            if ($isDryRun) {
+                                $this->warn("  ⚠ Would update customer ID from {$subscription->stripe_customer_id} to {$foundResult['customer_id']}");
+                            } elseif ($shouldFix) {
+                                $subscription->update(['stripe_customer_id' => $foundResult['customer_id']]);
+                                $this->info("  ✓ Updated customer ID to {$foundResult['customer_id']}");
+                            }
+                        }
+                    }
+                }
+
+                if ($foundResult && isset($foundResult['subscription'])) {
+                    // Check if it's an active/past_due subscription
+                    if (in_array($foundResult['status'], ['active', 'past_due'])) {
+                        // Found an active subscription, handle update
+                        $result = $this->handleFoundSubscription(
+                            $team,
+                            $subscription,
+                            $foundResult['subscription'],
+                            $searchMethod,
+                            $isDryRun,
+                            $shouldFix,
+                            $stats
+                        );
+
+                        fputcsv($out, [
+                            $team->id,
+                            $team->name,
+                            $subscription->stripe_customer_id,
+                            $result['id'],
+                            $dbStatus,
+                            $result['status'],
+                            $result['action'],
+                            implode(', ', $memberEmails),
+                            $subscription->stripe_customer_id ? "https://dashboard.stripe.com/customers/{$subscription->stripe_customer_id}" : 'N/A',
+                            $result['url'],
+                        ]);
+                    } else {
+                        // Found subscription but it's canceled/expired - needs to be deactivated
+                        $this->warn("  → Found {$foundResult['status']} subscription {$foundResult['subscription']->id} - needs deactivation");
+
+                        $result = $this->handleMissingSubscription($team, $subscription, $foundResult['status'], $isDryRun, $shouldFix, $stats);
+
+                        fputcsv($out, [
+                            $team->id,
+                            $team->name,
+                            $subscription->stripe_customer_id,
+                            $foundResult['subscription']->id,
+                            $dbStatus,
+                            $foundResult['status'],
+                            'needs_fix',
+                            implode(', ', $memberEmails),
+                            $subscription->stripe_customer_id ? "https://dashboard.stripe.com/customers/{$subscription->stripe_customer_id}" : 'N/A',
+                            "https://dashboard.stripe.com/subscriptions/{$foundResult['subscription']->id}",
+                        ]);
+                    }
+                } else {
+                    // No subscription found at all
+                    $this->line('  → No subscription found');
+
+                    $stripeStatus = 'not_found';
+                    $result = $this->handleMissingSubscription($team, $subscription, $stripeStatus, $isDryRun, $shouldFix, $stats);
+
+                    fputcsv($out, [
+                        $team->id,
+                        $team->name,
+                        $subscription->stripe_customer_id,
+                        'N/A',
+                        $dbStatus,
+                        $result['status'],
+                        $result['action'],
+                        implode(', ', $memberEmails),
+                        $subscription->stripe_customer_id ? "https://dashboard.stripe.com/customers/{$subscription->stripe_customer_id}" : 'N/A',
+                        'N/A',
+                    ]);
+                }
+            } else {
+                // First validate the subscription ID format
+                if (! str_starts_with($subscription->stripe_subscription_id, 'sub_')) {
+                    $this->warn("  ⚠ Invalid subscription ID format (doesn't start with 'sub_')");
+                }
+
+                try {
+                    $stripeSubscription = $stripe->subscriptions->retrieve(
+                        $subscription->stripe_subscription_id
+                    );
+
+                    $stripeStatus = $stripeSubscription->status;
+
+                    // Determine if action is needed
+                    switch ($stripeStatus) {
+                        case 'active':
+                            $stats['valid_active']++;
+                            $action = 'valid';
+                            break;
+
+                        case 'past_due':
+                            $stats['valid_past_due']++;
+                            $action = 'valid';
+                            // Ensure past_due flag is set
+                            if (! $subscription->stripe_past_due) {
+                                if ($isDryRun) {
+                                    $this->info("Would set stripe_past_due=true for Team {$team->id}");
+                                } elseif ($shouldFix) {
+                                    $subscription->update(['stripe_past_due' => true]);
+                                }
+                            }
+                            break;
+
+                        case 'canceled':
+                        case 'incomplete_expired':
+                        case 'unpaid':
+                        case 'incomplete':
+                            $stats['canceled']++;
+                            $action = 'needs_fix';
+
+                            // Only output problematic subscriptions
+                            fputcsv($out, [
+                                $team->id,
+                                $team->name,
+                                $subscription->stripe_customer_id,
+                                $subscription->stripe_subscription_id,
+                                $dbStatus,
+                                $stripeStatus,
+                                $action,
+                                implode(', ', $memberEmails),
+                                "https://dashboard.stripe.com/customers/{$subscription->stripe_customer_id}",
+                                "https://dashboard.stripe.com/subscriptions/{$subscription->stripe_subscription_id}",
+                            ]);
+
+                            if ($isDryRun) {
+                                $this->info("Would deactivate subscription for Team {$team->id} - status: {$stripeStatus}");
+                            } elseif ($shouldFix) {
+                                $this->fixSubscription($team, $subscription, $stripeStatus);
+                                $stats['fixed']++;
+                            }
+                            break;
+
+                        default:
+                            $stats['invalid']++;
+                            $action = 'unknown';
+
+                            // Only output problematic subscriptions
+                            fputcsv($out, [
+                                $team->id,
+                                $team->name,
+                                $subscription->stripe_customer_id,
+                                $subscription->stripe_subscription_id,
+                                $dbStatus,
+                                $stripeStatus,
+                                $action,
+                                implode(', ', $memberEmails),
+                                "https://dashboard.stripe.com/customers/{$subscription->stripe_customer_id}",
+                                "https://dashboard.stripe.com/subscriptions/{$subscription->stripe_subscription_id}",
+                            ]);
+                            break;
+                    }
+
+                } catch (InvalidRequestException $e) {
+                    $this->error('  → Error: '.$e->getMessage());
+
+                    if ($e->getStripeCode() === 'resource_missing' || $e->getHttpStatus() === 404) {
+                        // Subscription doesn't exist, try to find by customer ID
+                        $this->warn("  → Subscription not found, checking customer's subscriptions...");
+
+                        $foundResult = null;
+                        if ($subscription->stripe_customer_id) {
+                            $foundResult = $this->searchSubscriptionsByCustomer($stripe, $subscription->stripe_customer_id);
+                        }
+
+                        if ($foundResult && isset($foundResult['subscription']) && in_array($foundResult['status'], ['active', 'past_due'])) {
+                            // Found an active subscription with different ID
+                            $this->warn("  → ID mismatch! DB: {$subscription->stripe_subscription_id}, Stripe: {$foundResult['subscription']->id}");
+
+                            fputcsv($out, [
+                                $team->id,
+                                $team->name,
+                                $subscription->stripe_customer_id,
+                                "WRONG ID: {$subscription->stripe_subscription_id} → {$foundResult['subscription']->id}",
+                                $dbStatus,
+                                $foundResult['status'],
+                                'id_mismatch',
+                                implode(', ', $memberEmails),
+                                "https://dashboard.stripe.com/customers/{$subscription->stripe_customer_id}",
+                                "https://dashboard.stripe.com/subscriptions/{$foundResult['subscription']->id}",
+                            ]);
+
+                            if ($isDryRun) {
+                                $this->warn("  → Would update subscription ID to {$foundResult['subscription']->id}");
+                            } elseif ($shouldFix) {
+                                $subscription->update([
+                                    'stripe_subscription_id' => $foundResult['subscription']->id,
+                                    'stripe_invoice_paid' => true,
+                                    'stripe_past_due' => $foundResult['status'] === 'past_due',
+                                ]);
+                                $stats['fixed']++;
+                                $this->info('  → Updated subscription ID');
+                            }
+
+                            $stats[$foundResult['status'] === 'active' ? 'valid_active' : 'valid_past_due']++;
+                        } else {
+                            // No active subscription found
+                            $stripeStatus = $foundResult ? $foundResult['status'] : 'not_found';
+                            $result = $this->handleMissingSubscription($team, $subscription, $stripeStatus, $isDryRun, $shouldFix, $stats);
+
+                            fputcsv($out, [
+                                $team->id,
+                                $team->name,
+                                $subscription->stripe_customer_id,
+                                $subscription->stripe_subscription_id,
+                                $dbStatus,
+                                $result['status'],
+                                $result['action'],
+                                implode(', ', $memberEmails),
+                                $subscription->stripe_customer_id ? "https://dashboard.stripe.com/customers/{$subscription->stripe_customer_id}" : 'N/A',
+                                $foundResult && isset($foundResult['subscription']) ? "https://dashboard.stripe.com/subscriptions/{$foundResult['subscription']->id}" : 'N/A',
+                            ]);
+                        }
+                    } else {
+                        // Other API error
+                        $stats['errors']++;
+                        $this->error('  → API Error - not marking as deleted');
+
+                        fputcsv($out, [
+                            $team->id,
+                            $team->name,
+                            $subscription->stripe_customer_id,
+                            $subscription->stripe_subscription_id,
+                            $dbStatus,
+                            'error: '.$e->getStripeCode(),
+                            'error',
+                            implode(', ', $memberEmails),
+                            $subscription->stripe_customer_id ? "https://dashboard.stripe.com/customers/{$subscription->stripe_customer_id}" : 'N/A',
+                            $subscription->stripe_subscription_id ? "https://dashboard.stripe.com/subscriptions/{$subscription->stripe_subscription_id}" : 'N/A',
+                        ]);
+                    }
+                } catch (\Exception $e) {
+                    $this->error('  → Unexpected error: '.$e->getMessage());
+                    $stats['errors']++;
+
+                    fputcsv($out, [
+                        $team->id,
+                        $team->name,
+                        $subscription->stripe_customer_id,
+                        $subscription->stripe_subscription_id,
+                        $dbStatus,
+                        'error',
+                        'error',
+                        implode(', ', $memberEmails),
+                        $subscription->stripe_customer_id ? "https://dashboard.stripe.com/customers/{$subscription->stripe_customer_id}" : 'N/A',
+                        $subscription->stripe_subscription_id ? "https://dashboard.stripe.com/subscriptions/{$subscription->stripe_subscription_id}" : 'N/A',
+                    ]);
+                }
+            }
+
+            $processedCount++;
+            if ($processedCount % 100 === 0) {
+                $this->info("Processed {$processedCount}/{$totalCount} subscriptions...");
+            }
+        }
+
+        fclose($out);
+
+        // Print summary
+        $this->newLine(2);
+        $this->info('=== Verification Summary ===');
+        $this->info("Total subscriptions checked: {$stats['total']}");
+        $this->newLine();
+
+        $this->info('Valid subscriptions in Stripe:');
+        $this->line("  - Active: {$stats['valid_active']}");
+        $this->line("  - Past Due: {$stats['valid_past_due']}");
+        $validTotal = $stats['valid_active'] + $stats['valid_past_due'];
+        $this->info("  Total valid: {$validTotal}");
+
+        $this->newLine();
+        $this->warn('Invalid subscriptions:');
+        $this->line("  - Canceled/Expired: {$stats['canceled']}");
+        $this->line("  - Missing/Not Found: {$stats['missing']}");
+        $this->line("  - Unknown status: {$stats['invalid']}");
+        $invalidTotal = $stats['canceled'] + $stats['missing'] + $stats['invalid'];
+        $this->warn("  Total invalid: {$invalidTotal}");
+
+        if ($stats['errors'] > 0) {
+            $this->newLine();
+            $this->error("Errors encountered: {$stats['errors']}");
+        }
+
+        if ($shouldFix && ! $isDryRun) {
+            $this->newLine();
+            $this->info("Fixed subscriptions: {$stats['fixed']}");
+        } elseif ($invalidTotal > 0 && ! $shouldFix) {
+            $this->newLine();
+            $this->comment('Run with --fix-verified to fix the discrepancies');
+        }
+
+        return 0;
+    }
+
+    /**
+     * Fix a subscription based on its status
+     */
+    private function fixSubscription($team, $subscription, $status)
+    {
+        $message = "Fixing subscription for Team ID: {$team->id} (Status: {$status})\n";
+        $message .= "Team Name: {$team->name}\n";
+        $message .= "Customer ID: {$subscription->stripe_customer_id}\n";
+        $message .= "Subscription ID: {$subscription->stripe_subscription_id}\n";
+
+        send_internal_notification($message);
+
+        // Call the team's subscription ended method which properly cleans up
+        $team->subscriptionEnded();
+    }
+
+    /**
+     * Search for subscriptions by customer ID
+     */
+    private function searchSubscriptionsByCustomer(StripeClient $stripe, $customerId, $requireActive = false)
+    {
+        try {
+            $subscriptions = $stripe->subscriptions->all([
+                'customer' => $customerId,
+                'limit' => 10,
+                'status' => 'all',
+            ]);
+
+            $this->line('  → Found '.count($subscriptions->data).' subscription(s) for customer');
+
+            // Look for active/past_due first
+            foreach ($subscriptions->data as $sub) {
+                $this->line("    - Subscription {$sub->id}: status={$sub->status}");
+                if (in_array($sub->status, ['active', 'past_due'])) {
+                    $this->info("    ✓ Found active/past_due subscription: {$sub->id}");
+
+                    return ['subscription' => $sub, 'status' => $sub->status, 'method' => 'customer_id'];
+                }
+            }
+
+            // If not requiring active and there are subscriptions, return first one
+            if (! $requireActive && count($subscriptions->data) > 0) {
+                $sub = $subscriptions->data[0];
+                $this->warn("    ⚠ Only found {$sub->status} subscription: {$sub->id}");
+
+                return ['subscription' => $sub, 'status' => $sub->status, 'method' => 'customer_id_first'];
+            }
+
+            return null;
+        } catch (\Exception $e) {
+            $this->error('  → Error searching by customer ID: '.$e->getMessage());
+
+            return null;
+        }
+    }
+
+    /**
+     * Search for subscriptions by team member emails
+     */
+    private function searchSubscriptionsByEmails(StripeClient $stripe, $emails)
+    {
+        $this->line('  → Searching by team member emails...');
+
+        foreach ($emails as $email) {
+            $this->line("    → Checking email: {$email}");
+
+            try {
+                $customers = $stripe->customers->all([
+                    'email' => $email,
+                    'limit' => 5,
+                ]);
+
+                if (count($customers->data) === 0) {
+                    $this->line('      - No customers found');
+
+                    continue;
+                }
+
+                $this->line('      - Found '.count($customers->data).' customer(s)');
+
+                foreach ($customers->data as $customer) {
+                    $this->line("      - Checking customer {$customer->id}");
+
+                    $result = $this->searchSubscriptionsByCustomer($stripe, $customer->id, true);
+                    if ($result) {
+                        $result['method'] = "email:{$email}";
+                        $result['customer_id'] = $customer->id;
+
+                        return $result;
+                    }
+                }
+            } catch (\Exception $e) {
+                $this->error("      - Error searching for email {$email}: ".$e->getMessage());
+            }
+        }
+
+        return null;
+    }
+
+    /**
+     * Handle found subscription update (only for active/past_due subscriptions)
+     */
+    private function handleFoundSubscription($team, $subscription, $foundSub, $searchMethod, $isDryRun, $shouldFix, &$stats)
+    {
+        $stripeStatus = $foundSub->status;
+        $this->info("  ✓ FOUND active/past_due subscription {$foundSub->id} (status: {$stripeStatus})");
+
+        // Only update if it's active or past_due
+        if (! in_array($stripeStatus, ['active', 'past_due'])) {
+            $this->error("  ERROR: handleFoundSubscription called with {$stripeStatus} subscription!");
+
+            return [
+                'id' => $foundSub->id,
+                'status' => $stripeStatus,
+                'action' => 'error',
+                'url' => "https://dashboard.stripe.com/subscriptions/{$foundSub->id}",
+            ];
+        }
+
+        if ($isDryRun) {
+            $this->warn("  → Would update subscription ID to {$foundSub->id} (status: {$stripeStatus})");
+        } elseif ($shouldFix) {
+            $subscription->update([
+                'stripe_subscription_id' => $foundSub->id,
+                'stripe_invoice_paid' => true,
+                'stripe_past_due' => $stripeStatus === 'past_due',
+            ]);
+            $stats['fixed']++;
+            $this->info("  → Updated subscription ID to {$foundSub->id}");
+        }
+
+        // Update stats
+        $stats[$stripeStatus === 'active' ? 'valid_active' : 'valid_past_due']++;
+
+        return [
+            'id' => "FOUND: {$foundSub->id}",
+            'status' => $stripeStatus,
+            'action' => "will_update (via {$searchMethod})",
+            'url' => "https://dashboard.stripe.com/subscriptions/{$foundSub->id}",
+        ];
+    }
+
+    /**
+     * Handle missing subscription
+     */
+    private function handleMissingSubscription($team, $subscription, $status, $isDryRun, $shouldFix, &$stats)
+    {
+        $stats['missing']++;
+
+        if ($isDryRun) {
+            $statusMsg = $status !== 'not_found' ? "status: {$status}" : 'no subscription found in Stripe';
+            $this->warn("  → Would deactivate subscription - {$statusMsg}");
+        } elseif ($shouldFix) {
+            $this->fixSubscription($team, $subscription, $status);
+            $stats['fixed']++;
+            $this->info('  → Deactivated subscription');
+        }
+
+        return [
+            'id' => 'N/A',
+            'status' => $status,
+            'action' => 'needs_fix',
+            'url' => 'N/A',
+        ];
+    }
+}

app/Console/Commands/Cloud/RestoreDatabase.php

@@ -0,0 +1,219 @@
+<?php
+
+namespace App\Console\Commands\Cloud;
+
+use Illuminate\Console\Command;
+
+class RestoreDatabase extends Command
+{
+    protected $signature = 'cloud:restore-database {file : Path to the database dump file} {--debug : Show detailed debug output}';
+
+    protected $description = 'Restore a PostgreSQL database from a dump file (development mode only)';
+
+    private bool $debug = false;
+
+    public function handle(): int
+    {
+        $this->debug = $this->option('debug');
+
+        if (! $this->isDevelopment()) {
+            $this->error('This command can only be run in development mode.');
+
+            return 1;
+        }
+
+        $filePath = $this->argument('file');
+
+        if (! file_exists($filePath)) {
+            $this->error("File not found: {$filePath}");
+
+            return 1;
+        }
+
+        if (! is_readable($filePath)) {
+            $this->error("File is not readable: {$filePath}");
+
+            return 1;
+        }
+
+        try {
+            $this->info('Starting database restoration...');
+
+            $database = config('database.connections.pgsql.database');
+            $host = config('database.connections.pgsql.host');
+            $port = config('database.connections.pgsql.port');
+            $username = config('database.connections.pgsql.username');
+            $password = config('database.connections.pgsql.password');
+
+            if (! $database || ! $username) {
+                $this->error('Database configuration is incomplete.');
+
+                return 1;
+            }
+
+            $this->info("Restoring to database: {$database}");
+
+            // Drop all tables
+            if (! $this->dropAllTables($database, $host, $port, $username, $password)) {
+                return 1;
+            }
+
+            // Restore the database dump
+            if (! $this->restoreDatabaseDump($filePath, $database, $host, $port, $username, $password)) {
+                return 1;
+            }
+
+            $this->info('Database restoration completed successfully!');
+
+            return 0;
+        } catch (\Exception $e) {
+            $this->error("An error occurred: {$e->getMessage()}");
+
+            return 1;
+        }
+    }
+
+    private function dropAllTables(string $database, string $host, string $port, string $username, string $password): bool
+    {
+        $this->info('Dropping all tables...');
+
+        // SQL to drop all tables
+        $dropTablesSQL = <<<'SQL'
+            DO $$ DECLARE
+                r RECORD;
+            BEGIN
+                FOR r IN (SELECT tablename FROM pg_tables WHERE schemaname = 'public') LOOP
+                    EXECUTE 'DROP TABLE IF EXISTS ' || quote_ident(r.tablename) || ' CASCADE';
+                END LOOP;
+            END $$;
+            SQL;
+
+        // Build the psql command to drop all tables
+        $command = sprintf(
+            'PGPASSWORD=%s psql -h %s -p %s -U %s -d %s -c %s',
+            escapeshellarg($password),
+            escapeshellarg($host),
+            escapeshellarg($port),
+            escapeshellarg($username),
+            escapeshellarg($database),
+            escapeshellarg($dropTablesSQL)
+        );
+
+        if ($this->debug) {
+            $this->line('<comment>Executing drop command:</comment>');
+            $this->line($command);
+        }
+
+        $output = shell_exec($command.' 2>&1');
+
+        if ($this->debug) {
+            $this->line("<comment>Output:</comment> {$output}");
+        }
+
+        $this->info('All tables dropped successfully.');
+
+        return true;
+    }
+
+    private function restoreDatabaseDump(string $filePath, string $database, string $host, string $port, string $username, string $password): bool
+    {
+        $this->info('Restoring database from dump file...');
+
+        // Handle gzipped files by decompressing first
+        $actualFile = $filePath;
+        if (str_ends_with($filePath, '.gz')) {
+            $actualFile = rtrim($filePath, '.gz');
+            $this->info('Decompressing gzipped dump file...');
+
+            $decompressCommand = sprintf(
+                'gunzip -c %s > %s',
+                escapeshellarg($filePath),
+                escapeshellarg($actualFile)
+            );
+
+            if ($this->debug) {
+                $this->line('<comment>Executing decompress command:</comment>');
+                $this->line($decompressCommand);
+            }
+
+            $decompressOutput = shell_exec($decompressCommand.' 2>&1');
+            if ($this->debug && $decompressOutput) {
+                $this->line("<comment>Decompress output:</comment> {$decompressOutput}");
+            }
+        }
+
+        // Use pg_restore for custom format dumps
+        $command = sprintf(
+            'PGPASSWORD=%s pg_restore -h %s -p %s -U %s -d %s -v %s',
+            escapeshellarg($password),
+            escapeshellarg($host),
+            escapeshellarg($port),
+            escapeshellarg($username),
+            escapeshellarg($database),
+            escapeshellarg($actualFile)
+        );
+
+        if ($this->debug) {
+            $this->line('<comment>Executing restore command:</comment>');
+            $this->line($command);
+        }
+
+        // Execute the restore command
+        $process = proc_open(
+            $command,
+            [
+                1 => ['pipe', 'w'],
+                2 => ['pipe', 'w'],
+            ],
+            $pipes
+        );
+
+        if (! is_resource($process)) {
+            $this->error('Failed to start restoration process.');
+
+            return false;
+        }
+
+        $output = stream_get_contents($pipes[1]);
+        $error = stream_get_contents($pipes[2]);
+        $exitCode = proc_close($process);
+
+        // Clean up decompressed file if we created one
+        if ($actualFile !== $filePath && file_exists($actualFile)) {
+            unlink($actualFile);
+        }
+
+        if ($this->debug) {
+            if ($output) {
+                $this->line('<comment>Output:</comment>');
+                $this->line($output);
+            }
+            if ($error) {
+                $this->line('<comment>Error output:</comment>');
+                $this->line($error);
+            }
+            $this->line("<comment>Exit code:</comment> {$exitCode}");
+        }
+
+        if ($exitCode !== 0) {
+            $this->error("Restoration failed with exit code: {$exitCode}");
+            if ($error) {
+                $this->error('Error details:');
+                $this->error($error);
+            }
+
+            return false;
+        }
+
+        if ($output && ! $this->debug) {
+            $this->line($output);
+        }
+
+        return true;
+    }
+
+    private function isDevelopment(): bool
+    {
+        return app()->environment(['local', 'development', 'dev']);
+    }
+}

app/Console/Commands/Cloud/SyncStripeSubscriptions.php

@@ -0,0 +1,101 @@
+<?php
+
+namespace App\Console\Commands\Cloud;
+
+use App\Jobs\SyncStripeSubscriptionsJob;
+use Illuminate\Console\Command;
+
+class SyncStripeSubscriptions extends Command
+{
+    protected $signature = 'cloud:sync-stripe-subscriptions {--fix : Actually fix discrepancies (default is check only)}';
+
+    protected $description = 'Sync subscription status with Stripe. By default only checks, use --fix to apply changes.';
+
+    public function handle(): int
+    {
+        if (! isCloud()) {
+            $this->error('This command can only be run on Coolify Cloud.');
+
+            return 1;
+        }
+
+        if (! isStripe()) {
+            $this->error('Stripe is not configured.');
+
+            return 1;
+        }
+
+        $fix = $this->option('fix');
+
+        if ($fix) {
+            $this->warn('Running with --fix: discrepancies will be corrected.');
+        } else {
+            $this->info('Running in check mode (no changes will be made). Use --fix to apply corrections.');
+        }
+
+        $this->newLine();
+
+        $job = new SyncStripeSubscriptionsJob($fix);
+        $fetched = 0;
+        $result = $job->handle(function (int $count) use (&$fetched): void {
+            $fetched = $count;
+            $this->output->write("\r  Fetching subscriptions from Stripe... {$fetched}");
+        });
+        if ($fetched > 0) {
+            $this->output->write("\r".str_repeat(' ', 60)."\r");
+        }
+
+        if (isset($result['error'])) {
+            $this->error($result['error']);
+
+            return 1;
+        }
+
+        $this->info("Total subscriptions checked: {$result['total_checked']}");
+        $this->newLine();
+
+        if (count($result['discrepancies']) > 0) {
+            $this->warn('Discrepancies found: '.count($result['discrepancies']));
+            $this->newLine();
+
+            foreach ($result['discrepancies'] as $discrepancy) {
+                $this->line("  - Subscription ID: {$discrepancy['subscription_id']}");
+                $this->line("    Team ID: {$discrepancy['team_id']}");
+                $this->line("    Stripe ID: {$discrepancy['stripe_subscription_id']}");
+                $this->line("    Stripe Status: {$discrepancy['stripe_status']}");
+                $this->newLine();
+            }
+
+            if ($fix) {
+                $this->info('All discrepancies have been fixed.');
+            } else {
+                $this->comment('Run with --fix to correct these discrepancies.');
+            }
+        } else {
+            $this->info('No discrepancies found. All subscriptions are in sync.');
+        }
+
+        if (count($result['resubscribed']) > 0) {
+            $this->newLine();
+            $this->warn('Resubscribed users (same email, different customer): '.count($result['resubscribed']));
+            $this->newLine();
+
+            foreach ($result['resubscribed'] as $resub) {
+                $this->line("  - Team ID: {$resub['team_id']} | Email: {$resub['email']}");
+                $this->line("    Old: {$resub['old_stripe_subscription_id']} (cus: {$resub['old_stripe_customer_id']})");
+                $this->line("    New: {$resub['new_stripe_subscription_id']} (cus: {$resub['new_stripe_customer_id']}) [{$resub['new_status']}]");
+                $this->newLine();
+            }
+        }
+
+        if (count($result['errors']) > 0) {
+            $this->newLine();
+            $this->error('Errors encountered: '.count($result['errors']));
+            foreach ($result['errors'] as $error) {
+                $this->line("  - Subscription {$error['subscription_id']}: {$error['error']}");
+            }
+        }
+
+        return 0;
+    }
+}

app/Console/Commands/CloudCheckSubscription.php

@@ -1,49 +0,0 @@
-<?php
-
-namespace App\Console\Commands;
-
-use App\Models\Team;
-use Illuminate\Console\Command;
-
-class CloudCheckSubscription extends Command
-{
-    /**
-     * The name and signature of the console command.
-     *
-     * @var string
-     */
-    protected $signature = 'cloud:check-subscription';
-
-    /**
-     * The console command description.
-     *
-     * @var string
-     */
-    protected $description = 'Check Cloud subscriptions';
-
-    /**
-     * Execute the console command.
-     */
-    public function handle()
-    {
-        $stripe = new \Stripe\StripeClient(config('subscription.stripe_api_key'));
-        $activeSubscribers = Team::whereRelation('subscription', 'stripe_invoice_paid', true)->get();
-        foreach ($activeSubscribers as $team) {
-            $stripeSubscriptionId = $team->subscription->stripe_subscription_id;
-            $stripeInvoicePaid = $team->subscription->stripe_invoice_paid;
-            $stripeCustomerId = $team->subscription->stripe_customer_id;
-            if (! $stripeSubscriptionId) {
-                echo "Team {$team->id} has no subscription, but invoice status is: {$stripeInvoicePaid}\n";
-                echo "Link on Stripe: https://dashboard.stripe.com/customers/{$stripeCustomerId}\n";
-
-                continue;
-            }
-            $subscription = $stripe->subscriptions->retrieve($stripeSubscriptionId);
-            if ($subscription->status === 'active') {
-                continue;
-            }
-            echo "Subscription {$stripeSubscriptionId} is not active ({$subscription->status})\n";
-            echo "Link on Stripe: https://dashboard.stripe.com/subscriptions/{$stripeSubscriptionId}\n";
-        }
-    }
-}

app/Console/Commands/CloudCleanupSubscriptions.php

@@ -1,101 +0,0 @@
-<?php
-
-namespace App\Console\Commands;
-
-use App\Events\ServerReachabilityChanged;
-use App\Models\Team;
-use Illuminate\Console\Command;
-
-class CloudCleanupSubscriptions extends Command
-{
-    protected $signature = 'cloud:cleanup-subs';
-
-    protected $description = 'Cleanup subcriptions teams';
-
-    public function handle()
-    {
-        try {
-            if (! isCloud()) {
-                $this->error('This command can only be run on cloud');
-
-                return;
-            }
-            $this->info('Cleaning up subcriptions teams');
-            $stripe = new \Stripe\StripeClient(config('subscription.stripe_api_key'));
-
-            $teams = Team::all()->filter(function ($team) {
-                return $team->id !== 0;
-            })->sortBy('id');
-            foreach ($teams as $team) {
-                if ($team) {
-                    $this->info("Checking team {$team->id}");
-                }
-                if (! data_get($team, 'subscription')) {
-                    $this->disableServers($team);
-
-                    continue;
-                }
-                // If the team has no subscription id and the invoice is paid, we need to reset the invoice paid status
-                if (! (data_get($team, 'subscription.stripe_subscription_id'))) {
-                    $this->info("Resetting invoice paid status for team {$team->id}");
-
-                    $team->subscription->update([
-                        'stripe_invoice_paid' => false,
-                        'stripe_trial_already_ended' => false,
-                        'stripe_subscription_id' => null,
-                    ]);
-                    $this->disableServers($team);
-
-                    continue;
-                } else {
-                    $subscription = $stripe->subscriptions->retrieve(data_get($team, 'subscription.stripe_subscription_id'), []);
-                    $status = data_get($subscription, 'status');
-                    if ($status === 'active') {
-                        $team->subscription->update([
-                            'stripe_invoice_paid' => true,
-                            'stripe_trial_already_ended' => false,
-                        ]);
-
-                        continue;
-                    }
-                    $this->info('Subscription status: '.$status);
-                    $this->info('Subscription id: '.data_get($team, 'subscription.stripe_subscription_id'));
-                    $confirm = $this->confirm('Do you want to cancel the subscription?', true);
-                    if (! $confirm) {
-                        $this->info("Skipping team {$team->id}");
-                    } else {
-                        $this->info("Cancelling subscription for team {$team->id}");
-                        $team->subscription->update([
-                            'stripe_invoice_paid' => false,
-                            'stripe_trial_already_ended' => false,
-                            'stripe_subscription_id' => null,
-                        ]);
-                        $this->disableServers($team);
-                    }
-                }
-            }
-        } catch (\Exception $e) {
-            $this->error($e->getMessage());
-
-            return;
-        }
-    }
-
-    private function disableServers(Team $team)
-    {
-        foreach ($team->servers as $server) {
-            if ($server->settings->is_usable === true || $server->settings->is_reachable === true || $server->ip !== '1.2.3.4') {
-                $this->info("Disabling server {$server->id} {$server->name}");
-                $server->settings()->update([
-                    'is_usable' => false,
-                    'is_reachable' => false,
-                ]);
-                $server->update([
-                    'ip' => '1.2.3.4',
-                ]);
-
-                ServerReachabilityChanged::dispatch($server);
-            }
-        }
-    }
-}

app/Console/Commands/Dev.php

@@ -2,7 +2,11 @@
 
 namespace App\Console\Commands;
 
+use App\Jobs\CheckHelperImageJob;
 use App\Models\InstanceSettings;
+use App\Models\ScheduledDatabaseBackupExecution;
+use App\Models\ScheduledTaskExecution;
+use Carbon\Carbon;
 use Illuminate\Console\Command;
 use Illuminate\Support\Facades\Artisan;
 
@@ -26,23 +30,62 @@ class Dev extends Command
         // Generate APP_KEY if not exists
 
         if (empty(config('app.key'))) {
-            echo "Generating APP_KEY.\n";
+            echo "   INFO  Generating APP_KEY.\n";
             Artisan::call('key:generate');
         }
 
         // Generate STORAGE link if not exists
         if (! file_exists(public_path('storage'))) {
-            echo "Generating STORAGE link.\n";
+            echo "   INFO  Generating storage link.\n";
             Artisan::call('storage:link');
         }
 
         // Seed database if it's empty
         $settings = InstanceSettings::find(0);
         if (! $settings) {
-            echo "Initializing instance, seeding database.\n";
+            echo "   INFO  Initializing instance, seeding database.\n";
             Artisan::call('migrate --seed');
         } else {
-            echo "Instance already initialized.\n";
+            echo "   INFO  Instance already initialized.\n";
         }
+
+        // Clean up stuck jobs and stale locks on development startup
+        try {
+            echo "   INFO  Cleaning up Redis (stuck jobs and stale locks)...\n";
+            Artisan::call('cleanup:redis', ['--restart' => true, '--clear-locks' => true]);
+            echo "   INFO  Redis cleanup completed.\n";
+        } catch (\Throwable $e) {
+            echo "   ERROR  Redis cleanup failed: {$e->getMessage()}\n";
+        }
+
+        try {
+            $updatedTaskCount = ScheduledTaskExecution::where('status', 'running')->update([
+                'status' => 'failed',
+                'message' => 'Marked as failed during Coolify startup - job was interrupted',
+                'finished_at' => Carbon::now(),
+            ]);
+
+            if ($updatedTaskCount > 0) {
+                echo "   INFO  Marked {$updatedTaskCount} stuck scheduled task executions as failed.\n";
+            }
+        } catch (\Throwable $e) {
+            echo "   ERROR  Could not clean up stuck scheduled task executions: {$e->getMessage()}\n";
+        }
+
+        try {
+            $updatedBackupCount = ScheduledDatabaseBackupExecution::where('status', 'running')->update([
+                'status' => 'failed',
+                'message' => 'Marked as failed during Coolify startup - job was interrupted',
+                'finished_at' => Carbon::now(),
+            ]);
+
+            if ($updatedBackupCount > 0) {
+                echo "   INFO  Marked {$updatedBackupCount} stuck database backup executions as failed.\n";
+            }
+        } catch (\Throwable $e) {
+            echo "   ERROR  Could not clean up stuck database backup executions: {$e->getMessage()}\n";
+        }
+
+        CheckHelperImageJob::dispatch();
     }
 }

app/Console/Commands/Emails.php

@@ -18,6 +18,7 @@ use Exception;
 use Illuminate\Console\Command;
 use Illuminate\Mail\Message;
 use Illuminate\Notifications\Messages\MailMessage;
+use Illuminate\Support\Str;
 use Mail;
 
 use function Laravel\Prompts\confirm;
@@ -167,7 +168,7 @@ class Emails extends Command
                     ]);
                 }
                 $output = 'Because of an error, the backup of the database '.$db->name.' failed.';
-                $this->mail = (new BackupFailed($backup, $db, $output))->toMail();
+                $this->mail = (new BackupFailed($backup, $db, $output, $backup->database_name ?? 'unknown'))->toMail();
                 $this->sendEmail();
                 break;
             case 'backup-success':

app/Console/Commands/Generate/OpenApi.php

@@ -32,7 +32,8 @@ class OpenApi extends Command
         echo $process->output();
 
         $yaml = file_get_contents('openapi.yaml');
-        $json = json_encode(Yaml::parse($yaml), JSON_PRETTY_PRINT);
+        
+        $json = json_encode(Yaml::parse($yaml), JSON_PRETTY_PRINT)."\n";
         file_put_contents('openapi.json', $json);
         echo "Converted OpenAPI YAML to JSON.\n";
     }

app/Console/Commands/Generate/Services.php

@@ -4,6 +4,7 @@ namespace App\Console\Commands\Generate;
 
 use Illuminate\Console\Command;
 use Illuminate\Support\Arr;
+use Illuminate\Support\Facades\Process;
 use Symfony\Component\Yaml\Yaml;
 
 class Services extends Command
@@ -16,7 +17,7 @@ class Services extends Command
     /**
      * {@inheritdoc}
      */
-    protected $description = 'Generate service-templates.yaml based on /templates/compose directory';
+    protected $description = 'Generates service-templates json file based on /templates/compose directory';
 
     public function handle(): int
     {
@@ -33,7 +34,10 @@ class Services extends Command
                 ];
             })->toJson(JSON_PRETTY_PRINT | JSON_UNESCAPED_SLASHES);
 
-        file_put_contents(base_path('templates/service-templates.json'), $serviceTemplatesJson.PHP_EOL);
+        file_put_contents(base_path('templates/'.config('constants.services.file_name')), $serviceTemplatesJson.PHP_EOL);
+
+        // Generate service-templates.json with SERVICE_URL changed to SERVICE_FQDN
+        $this->generateServiceTemplatesWithFqdn();
 
         return self::SUCCESS;
     }
@@ -71,8 +75,10 @@ class Services extends Command
             'slogan' => $data->get('slogan', str($file)->headline()),
             'compose' => $compose,
             'tags' => $tags,
+            'category' => $data->get('category'),
             'logo' => $data->get('logo', 'svgs/default.webp'),
             'minversion' => $data->get('minversion', '0.0.0'),
+            'template_last_updated_at' => $this->templateLastUpdatedAt($file),
         ];
 
         if ($port = $data->get('port')) {
@@ -84,6 +90,193 @@ class Services extends Command
             $payload['envs'] = base64_encode($envFileContent);
         }
 
+        if (str($data->get('amd_only'))->toBoolean()) {
+            $payload['amd_only'] = true;
+        }
+
+        if (str($data->get('arm_only'))->toBoolean()) {
+            $payload['arm_only'] = true;
+        }
+
+        return $payload;
+    }
+
+    private function templateLastUpdatedAt(string $file): ?string
+    {
+        $process = Process::path(base_path())->run([
+            'git',
+            'log',
+            '-1',
+            '--format=%cI',
+            '--',
+            "templates/compose/{$file}",
+        ]);
+
+        if ($process->failed()) {
+            return null;
+        }
+
+        $timestamp = trim($process->output());
+
+        return $timestamp === '' ? null : $timestamp;
+    }
+
+    private function generateServiceTemplatesWithFqdn(): void
+    {
+        $serviceTemplatesWithFqdn = collect(array_merge(
+            glob(base_path('templates/compose/*.yaml')),
+            glob(base_path('templates/compose/*.yml'))
+        ))
+            ->mapWithKeys(function ($file): array {
+                $file = basename($file);
+                $parsed = $this->processFileWithFqdn($file);
+
+                return $parsed === false ? [] : [
+                    Arr::pull($parsed, 'name') => $parsed,
+                ];
+            })->toJson(JSON_PRETTY_PRINT | JSON_UNESCAPED_SLASHES);
+
+        file_put_contents(base_path('templates/service-templates.json'), $serviceTemplatesWithFqdn.PHP_EOL);
+
+        // Generate service-templates-raw.json with non-base64 encoded compose content
+        // $this->generateServiceTemplatesRaw();
+    }
+
+    private function processFileWithFqdn(string $file): false|array
+    {
+        $content = file_get_contents(base_path("templates/compose/$file"));
+
+        $data = collect(explode(PHP_EOL, $content))->mapWithKeys(function ($line): array {
+            preg_match('/^#(?<key>.*):(?<value>.*)$/U', $line, $m);
+
+            return $m ? [trim($m['key']) => trim($m['value'])] : [];
+        });
+
+        if (str($data->get('ignore'))->toBoolean()) {
+            return false;
+        }
+
+        $documentation = $data->get('documentation');
+        $documentation = $documentation ? $documentation.'?utm_source=coolify.io' : 'https://coolify.io/docs';
+
+        // Replace SERVICE_URL with SERVICE_FQDN in the content
+        $modifiedContent = str_replace('SERVICE_URL', 'SERVICE_FQDN', $content);
+
+        $json = Yaml::parse($modifiedContent);
+        $compose = base64_encode(Yaml::dump($json, 10, 2));
+
+        $tags = str($data->get('tags'))->lower()->explode(',')->map(fn ($tag) => trim($tag))->filter();
+        $tags = $tags->isEmpty() ? null : $tags->all();
+
+        $payload = [
+            'name' => pathinfo($file, PATHINFO_FILENAME),
+            'documentation' => $documentation,
+            'slogan' => $data->get('slogan', str($file)->headline()),
+            'compose' => $compose,
+            'tags' => $tags,
+            'category' => $data->get('category'),
+            'logo' => $data->get('logo', 'svgs/default.webp'),
+            'minversion' => $data->get('minversion', '0.0.0'),
+            'template_last_updated_at' => $this->templateLastUpdatedAt($file),
+        ];
+
+        if ($port = $data->get('port')) {
+            $payload['port'] = $port;
+        }
+
+        if ($envFile = $data->get('env_file')) {
+            $envFileContent = file_get_contents(base_path("templates/compose/$envFile"));
+            // Also replace SERVICE_URL with SERVICE_FQDN in env file content
+            $modifiedEnvContent = str_replace('SERVICE_URL', 'SERVICE_FQDN', $envFileContent);
+            $payload['envs'] = base64_encode($modifiedEnvContent);
+        }
+
+        if (str($data->get('amd_only'))->toBoolean()) {
+            $payload['amd_only'] = true;
+        }
+
+        if (str($data->get('arm_only'))->toBoolean()) {
+            $payload['arm_only'] = true;
+        }
+
+        return $payload;
+    }
+
+    private function generateServiceTemplatesRaw(): void
+    {
+        $serviceTemplatesRaw = collect(array_merge(
+            glob(base_path('templates/compose/*.yaml')),
+            glob(base_path('templates/compose/*.yml'))
+        ))
+            ->mapWithKeys(function ($file): array {
+                $file = basename($file);
+                $parsed = $this->processFileWithFqdnRaw($file);
+
+                return $parsed === false ? [] : [
+                    Arr::pull($parsed, 'name') => $parsed,
+                ];
+            })->toJson(JSON_PRETTY_PRINT | JSON_UNESCAPED_SLASHES);
+
+        file_put_contents(base_path('templates/service-templates-raw.json'), $serviceTemplatesRaw.PHP_EOL);
+    }
+
+    private function processFileWithFqdnRaw(string $file): false|array
+    {
+        $content = file_get_contents(base_path("templates/compose/$file"));
+
+        $data = collect(explode(PHP_EOL, $content))->mapWithKeys(function ($line): array {
+            preg_match('/^#(?<key>.*):(?<value>.*)$/U', $line, $m);
+
+            return $m ? [trim($m['key']) => trim($m['value'])] : [];
+        });
+
+        if (str($data->get('ignore'))->toBoolean()) {
+            return false;
+        }
+
+        $documentation = $data->get('documentation');
+        $documentation = $documentation ? $documentation.'?utm_source=coolify.io' : 'https://coolify.io/docs';
+
+        // Replace SERVICE_URL with SERVICE_FQDN in the content
+        $modifiedContent = str_replace('SERVICE_URL', 'SERVICE_FQDN', $content);
+
+        $json = Yaml::parse($modifiedContent);
+        $compose = Yaml::dump($json, 10, 2); // Not base64 encoded
+
+        $tags = str($data->get('tags'))->lower()->explode(',')->map(fn ($tag) => trim($tag))->filter();
+        $tags = $tags->isEmpty() ? null : $tags->all();
+
+        $payload = [
+            'name' => pathinfo($file, PATHINFO_FILENAME),
+            'documentation' => $documentation,
+            'slogan' => $data->get('slogan', str($file)->headline()),
+            'compose' => $compose,
+            'tags' => $tags,
+            'category' => $data->get('category'),
+            'logo' => $data->get('logo', 'svgs/default.webp'),
+            'minversion' => $data->get('minversion', '0.0.0'),
+            'template_last_updated_at' => $this->templateLastUpdatedAt($file),
+        ];
+
+        if ($port = $data->get('port')) {
+            $payload['port'] = $port;
+        }
+
+        if ($envFile = $data->get('env_file')) {
+            $envFileContent = file_get_contents(base_path("templates/compose/$envFile"));
+            // Also replace SERVICE_URL with SERVICE_FQDN in env file content (not base64 encoded)
+            $modifiedEnvContent = str_replace('SERVICE_URL', 'SERVICE_FQDN', $envFileContent);
+            $payload['envs'] = $modifiedEnvContent;
+        }
+
+        if (str($data->get('amd_only'))->toBoolean()) {
+            $payload['amd_only'] = true;
+        }
+
+        if (str($data->get('arm_only'))->toBoolean()) {
+            $payload['arm_only'] = true;
+        }
+
         return $payload;
     }
 }

app/Console/Commands/GenerateTestingSchema.php

@@ -0,0 +1,222 @@
+<?php
+
+namespace App\Console\Commands;
+
+use Illuminate\Console\Command;
+use Illuminate\Support\Facades\DB;
+
+class GenerateTestingSchema extends Command
+{
+    protected $signature = 'schema:generate-testing {--connection=pgsql : The database connection to read from}';
+
+    protected $description = 'Generate SQLite testing schema from the PostgreSQL database';
+
+    private array $typeMap = [
+        '/\bbigint\b/' => 'INTEGER',
+        '/\binteger\b/' => 'INTEGER',
+        '/\bsmallint\b/' => 'INTEGER',
+        '/\bboolean\b/' => 'INTEGER',
+        '/character varying\(\d+\)/' => 'TEXT',
+        '/timestamp\(\d+\) without time zone/' => 'TEXT',
+        '/timestamp\(\d+\) with time zone/' => 'TEXT',
+        '/\bjsonb\b/' => 'TEXT',
+        '/\bjson\b/' => 'TEXT',
+        '/\buuid\b/' => 'TEXT',
+        '/double precision/' => 'REAL',
+        '/numeric\(\d+,\d+\)/' => 'REAL',
+        '/\bdate\b/' => 'TEXT',
+    ];
+
+    private array $castRemovals = [
+        '::character varying',
+        '::text',
+        '::integer',
+        '::boolean',
+        '::timestamp without time zone',
+        '::timestamp with time zone',
+        '::numeric',
+    ];
+
+    public function handle(): int
+    {
+        $connection = $this->option('connection');
+
+        if (DB::connection($connection)->getDriverName() !== 'pgsql') {
+            $this->error("Connection '{$connection}' is not PostgreSQL.");
+
+            return self::FAILURE;
+        }
+
+        $this->info('Reading schema from PostgreSQL...');
+
+        $tables = $this->getTables($connection);
+        $lastMigration = DB::connection($connection)
+            ->table('migrations')
+            ->orderByDesc('id')
+            ->value('migration');
+
+        $output = [];
+        $output[] = '-- Generated by: php artisan schema:generate-testing';
+        $output[] = '-- Date: '.now()->format('Y-m-d H:i:s');
+        $output[] = '-- Last migration: '.($lastMigration ?? 'none');
+        $output[] = '';
+
+        foreach ($tables as $table) {
+            $columns = $this->getColumns($connection, $table);
+            $output[] = $this->generateCreateTable($table, $columns);
+        }
+
+        $indexes = $this->getIndexes($connection, $tables);
+        foreach ($indexes as $index) {
+            $output[] = $index;
+        }
+
+        $output[] = '';
+        $output[] = '-- Migration records';
+
+        $migrations = DB::connection($connection)->table('migrations')->orderBy('id')->get();
+        foreach ($migrations as $m) {
+            $migration = str_replace("'", "''", $m->migration);
+            $output[] = "INSERT INTO \"migrations\" (\"id\", \"migration\", \"batch\") VALUES ({$m->id}, '{$migration}', {$m->batch});";
+        }
+
+        $path = database_path('schema/testing-schema.sql');
+
+        if (! is_dir(dirname($path))) {
+            mkdir(dirname($path), 0755, true);
+        }
+
+        file_put_contents($path, implode("\n", $output)."\n");
+
+        $this->info("Schema written to {$path}");
+        $this->info(count($tables).' tables, '.count($migrations).' migration records.');
+
+        return self::SUCCESS;
+    }
+
+    private function getTables(string $connection): array
+    {
+        return collect(DB::connection($connection)->select(
+            "SELECT tablename FROM pg_tables WHERE schemaname = 'public' ORDER BY tablename"
+        ))->pluck('tablename')->toArray();
+    }
+
+    private function getColumns(string $connection, string $table): array
+    {
+        return DB::connection($connection)->select(
+            "SELECT column_name, data_type, character_maximum_length, column_default,
+                    is_nullable, udt_name, numeric_precision, numeric_scale, datetime_precision
+             FROM information_schema.columns
+             WHERE table_schema = 'public' AND table_name = ?
+             ORDER BY ordinal_position",
+            [$table]
+        );
+    }
+
+    private function generateCreateTable(string $table, array $columns): string
+    {
+        $lines = [];
+
+        foreach ($columns as $col) {
+            $lines[] = '    '.$this->generateColumnDef($table, $col);
+        }
+
+        return "CREATE TABLE IF NOT EXISTS \"{$table}\" (\n".implode(",\n", $lines)."\n);\n";
+    }
+
+    private function generateColumnDef(string $table, object $col): string
+    {
+        $name = $col->column_name;
+        $sqliteType = $this->convertType($col);
+
+        // Auto-increment primary key for id columns
+        if ($name === 'id' && $sqliteType === 'INTEGER' && $col->is_nullable === 'NO' && str_contains((string) $col->column_default, 'nextval')) {
+            return "\"{$name}\" INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL";
+        }
+
+        $parts = ["\"{$name}\"", $sqliteType];
+
+        // Default value
+        $default = $col->column_default;
+        if ($default !== null && ! str_contains($default, 'nextval')) {
+            $default = $this->cleanDefault($default);
+            $parts[] = "DEFAULT {$default}";
+        }
+
+        // NOT NULL
+        if ($col->is_nullable === 'NO') {
+            $parts[] = 'NOT NULL';
+        }
+
+        return implode(' ', $parts);
+    }
+
+    private function convertType(object $col): string
+    {
+        $pgType = $col->data_type;
+
+        return match (true) {
+            in_array($pgType, ['bigint', 'integer', 'smallint']) => 'INTEGER',
+            $pgType === 'boolean' => 'INTEGER',
+            in_array($pgType, ['character varying', 'text', 'USER-DEFINED']) => 'TEXT',
+            str_contains($pgType, 'timestamp') => 'TEXT',
+            in_array($pgType, ['json', 'jsonb']) => 'TEXT',
+            $pgType === 'uuid' => 'TEXT',
+            $pgType === 'double precision' => 'REAL',
+            $pgType === 'numeric' => 'REAL',
+            $pgType === 'date' => 'TEXT',
+            default => 'TEXT',
+        };
+    }
+
+    private function cleanDefault(string $default): string
+    {
+        foreach ($this->castRemovals as $cast) {
+            $default = str_replace($cast, '', $default);
+        }
+
+        // Remove array type casts like ::text[]
+        $default = preg_replace('/::[\w\s]+(\[\])?/', '', $default);
+
+        return $default;
+    }
+
+    private function getIndexes(string $connection, array $tables): array
+    {
+        $results = [];
+
+        $indexes = DB::connection($connection)->select(
+            "SELECT indexname, tablename, indexdef FROM pg_indexes
+             WHERE schemaname = 'public'
+             ORDER BY tablename, indexname"
+        );
+
+        foreach ($indexes as $idx) {
+            $def = $idx->indexdef;
+
+            // Skip primary key indexes
+            if (str_contains($def, '_pkey')) {
+                continue;
+            }
+
+            // Skip PG-specific indexes (GIN, GIST, expression indexes)
+            if (preg_match('/USING (gin|gist)/i', $def)) {
+                continue;
+            }
+            if (str_contains($def, '->>') || str_contains($def, '::')) {
+                continue;
+            }
+
+            // Convert to SQLite-compatible CREATE INDEX
+            $unique = str_contains($def, 'UNIQUE') ? 'UNIQUE ' : '';
+
+            // Extract columns from the index definition
+            if (preg_match('/\((.+)\)$/', $def, $m)) {
+                $cols = $m[1];
+                $results[] = "CREATE {$unique}INDEX IF NOT EXISTS \"{$idx->indexname}\" ON \"{$idx->tablename}\" ({$cols});";
+            }
+        }
+
+        return $results;
+    }
+}

app/Console/Commands/Horizon.php

@@ -1,23 +0,0 @@
-<?php
-
-namespace App\Console\Commands;
-
-use Illuminate\Console\Command;
-
-class Horizon extends Command
-{
-    protected $signature = 'start:horizon';
-
-    protected $description = 'Start Horizon';
-
-    public function handle()
-    {
-        if (config('constants.horizon.is_horizon_enabled')) {
-            $this->info('Horizon is enabled on this server.');
-            $this->call('horizon');
-            exit(0);
-        } else {
-            exit(0);
-        }
-    }
-}

app/Console/Commands/Init.php

@@ -5,12 +5,17 @@ namespace App\Console\Commands;
 use App\Enums\ActivityTypes;
 use App\Enums\ApplicationDeploymentStatus;
 use App\Jobs\CheckHelperImageJob;
+use App\Jobs\PullChangelog;
 use App\Models\ApplicationDeploymentQueue;
 use App\Models\Environment;
+use App\Models\InstanceSettings;
 use App\Models\ScheduledDatabaseBackup;
+use App\Models\ScheduledDatabaseBackupExecution;
+use App\Models\ScheduledTaskExecution;
 use App\Models\Server;
 use App\Models\StandalonePostgresql;
 use App\Models\User;
+use Carbon\Carbon;
 use Illuminate\Console\Command;
 use Illuminate\Support\Facades\Artisan;
 use Illuminate\Support\Facades\File;
@@ -18,27 +23,49 @@ use Illuminate\Support\Facades\Http;
 
 class Init extends Command
 {
-    protected $signature = 'app:init {--force-cloud}';
+    protected $signature = 'app:init';
 
     protected $description = 'Cleanup instance related stuffs';
 
     public $servers = null;
 
+    public InstanceSettings $settings;
+
     public function handle()
     {
-        $this->optimize();
+        Artisan::call('optimize:clear');
+        Artisan::call('optimize');
 
-        if (isCloud() && ! $this->option('force-cloud')) {
-            echo "Skipping init as we are on cloud and --force-cloud option is not set\n";
+        try {
+            $this->pullTemplatesFromCDN();
+        } catch (\Throwable $e) {
+            echo "Could not pull templates from CDN: {$e->getMessage()}\n";
+        }
 
+        try {
+            $this->pullChangelogFromGitHub();
+        } catch (\Throwable $e) {
+            echo "Could not changelogs from github: {$e->getMessage()}\n";
+        }
+
+        try {
+            $this->pullHelperImage();
+        } catch (\Throwable $e) {
+            echo "Error in pullHelperImage command: {$e->getMessage()}\n";
+        }
+
+        if (isCloud()) {
             return;
         }
 
+        $this->settings = instanceSettings();
         $this->servers = Server::all();
-        if (! isCloud()) {
+
+        $do_not_track = data_get($this->settings, 'do_not_track', true);
+        if ($do_not_track == false) {
             $this->sendAliveSignal();
-            get_public_ips();
         }
+        get_public_ips();
 
         // Backward compatibility
         $this->replaceSlashInEnvironmentName();
@@ -46,51 +73,83 @@ class Init extends Command
         $this->updateUserEmails();
         //
         $this->updateTraefikLabels();
-        if (! isCloud() || $this->option('force-cloud')) {
-            $this->cleanupUnusedNetworkFromCoolifyProxy();
-        }
-
-        $this->call('cleanup:redis');
-
-        $this->call('cleanup:stucked-resources');
+        $this->cleanupUnusedNetworkFromCoolifyProxy();
 
         try {
-            $this->pullHelperImage();
+            $this->call('cleanup:redis', ['--restart' => true, '--clear-locks' => true]);
         } catch (\Throwable $e) {
-            //
+            echo "Error in cleanup:redis command: {$e->getMessage()}\n";
         }
+        try {
+            $this->call('cleanup:names');
+        } catch (\Throwable $e) {
+            echo "Error in cleanup:names command: {$e->getMessage()}\n";
+        }
+        try {
+            $this->call('cleanup:stucked-resources');
+        } catch (\Throwable $e) {
+            echo "Error in cleanup:stucked-resources command: {$e->getMessage()}\n";
+            echo "Continuing with initialization - cleanup errors will not prevent Coolify from starting\n";
+        }
+        try {
+            $updatedCount = ApplicationDeploymentQueue::whereIn('status', [
+                ApplicationDeploymentStatus::IN_PROGRESS->value,
+                ApplicationDeploymentStatus::QUEUED->value,
+            ])->update([
+                'status' => ApplicationDeploymentStatus::FAILED->value,
+            ]);
 
-        if (isCloud()) {
-            try {
-                $this->cleanupUnnecessaryDynamicProxyConfiguration();
-                $this->pullTemplatesFromCDN();
-            } catch (\Throwable $e) {
-                echo "Could not pull templates from CDN: {$e->getMessage()}\n";
+            if ($updatedCount > 0) {
+                echo "Marked {$updatedCount} stuck deployments as failed\n";
             }
-
-            return;
+        } catch (\Throwable $e) {
+            echo "Could not cleanup inprogress deployments: {$e->getMessage()}\n";
         }
 
         try {
-            $this->cleanupInProgressApplicationDeployments();
-            $this->pullTemplatesFromCDN();
+            $updatedTaskCount = ScheduledTaskExecution::where('status', 'running')->update([
+                'status' => 'failed',
+                'message' => 'Marked as failed during Coolify startup - job was interrupted',
+                'finished_at' => Carbon::now(),
+            ]);
+
+            if ($updatedTaskCount > 0) {
+                echo "Marked {$updatedTaskCount} stuck scheduled task executions as failed\n";
+            }
         } catch (\Throwable $e) {
-            echo "Could not pull templates from CDN: {$e->getMessage()}\n";
+            echo "Could not cleanup stuck scheduled task executions: {$e->getMessage()}\n";
         }
+
+        try {
+            $updatedBackupCount = ScheduledDatabaseBackupExecution::where('status', 'running')->update([
+                'status' => 'failed',
+                'message' => 'Marked as failed during Coolify startup - job was interrupted',
+                'finished_at' => Carbon::now(),
+            ]);
+
+            if ($updatedBackupCount > 0) {
+                echo "Marked {$updatedBackupCount} stuck database backup executions as failed\n";
+            }
+        } catch (\Throwable $e) {
+            echo "Could not cleanup stuck database backup executions: {$e->getMessage()}\n";
+        }
+
         try {
             $localhost = $this->servers->where('id', 0)->first();
-            $localhost->setupDynamicProxyConfiguration();
+            if ($localhost) {
+                $localhost->setupDynamicProxyConfiguration();
+            }
         } catch (\Throwable $e) {
             echo "Could not setup dynamic configuration: {$e->getMessage()}\n";
         }
-        $settings = instanceSettings();
+
         if (! is_null(config('constants.coolify.autoupdate', null))) {
             if (config('constants.coolify.autoupdate') == true) {
                 echo "Enabling auto-update\n";
-                $settings->update(['is_auto_update_enabled' => true]);
+                $this->settings->update(['is_auto_update_enabled' => true]);
             } else {
                 echo "Disabling auto-update\n";
-                $settings->update(['is_auto_update_enabled' => false]);
+                $this->settings->update(['is_auto_update_enabled' => false]);
             }
         }
     }
@@ -105,21 +164,25 @@ class Init extends Command
         $response = Http::retry(3, 1000)->get(config('constants.services.official'));
         if ($response->successful()) {
             $services = $response->json();
-            File::put(base_path('templates/service-templates.json'), json_encode($services));
+            File::put(base_path('templates/'.config('constants.services.file_name')), json_encode($services));
         }
     }
 
-    private function optimize()
+    private function pullChangelogFromGitHub()
     {
-        Artisan::call('optimize:clear');
-        Artisan::call('optimize');
+        try {
+            PullChangelog::dispatch();
+            echo "Changelog fetch initiated\n";
+        } catch (\Throwable $e) {
+            echo "Could not fetch changelog from GitHub: {$e->getMessage()}\n";
+        }
     }
 
     private function updateUserEmails()
     {
         try {
             User::whereRaw('email ~ \'[A-Z]\'')->get()->each(function (User $user) {
-                $user->update(['email' => strtolower($user->email)]);
+                $user->update(['email' => $user->email]);
             });
         } catch (\Throwable $e) {
             echo "Error in updating user emails: {$e->getMessage()}\n";
@@ -135,27 +198,6 @@ class Init extends Command
         }
     }
 
-    private function cleanupUnnecessaryDynamicProxyConfiguration()
-    {
-        foreach ($this->servers as $server) {
-            try {
-                if (! $server->isFunctional()) {
-                    continue;
-                }
-                if ($server->id === 0) {
-                    continue;
-                }
-                $file = $server->proxyPath().'/dynamic/coolify.yaml';
-
-                return instant_remote_process([
-                    "rm -f $file",
-                ], $server, false);
-            } catch (\Throwable $e) {
-                echo "Error in cleaning up unnecessary dynamic proxy configuration: {$e->getMessage()}\n";
-            }
-        }
-    }
-
     private function cleanupUnusedNetworkFromCoolifyProxy()
     {
         foreach ($this->servers as $server) {
@@ -170,18 +212,19 @@ class Init extends Command
                 $removeNetworks = $allNetworks->diff($networks);
                 $commands = collect();
                 foreach ($removeNetworks as $network) {
-                    $out = instant_remote_process(["docker network inspect -f json $network | jq '.[].Containers | if . == {} then null else . end'"], $server, false);
+                    $safe = escapeshellarg($network);
+                    $out = instant_remote_process(["docker network inspect -f json {$safe} | jq '.[].Containers | if . == {} then null else . end'"], $server, false);
                     if (empty($out)) {
-                        $commands->push("docker network disconnect $network coolify-proxy >/dev/null 2>&1 || true");
-                        $commands->push("docker network rm $network >/dev/null 2>&1 || true");
+                        $commands->push("docker network disconnect {$safe} coolify-proxy >/dev/null 2>&1 || true");
+                        $commands->push("docker network rm {$safe} >/dev/null 2>&1 || true");
                     } else {
                         $data = collect(json_decode($out, true));
                         if ($data->count() === 1) {
                             // If only coolify-proxy itself is connected to that network (it should not be possible, but who knows)
                             $isCoolifyProxyItself = data_get($data->first(), 'Name') === 'coolify-proxy';
                             if ($isCoolifyProxyItself) {
-                                $commands->push("docker network disconnect $network coolify-proxy >/dev/null 2>&1 || true");
-                                $commands->push("docker network rm $network >/dev/null 2>&1 || true");
+                                $commands->push("docker network disconnect {$safe} coolify-proxy >/dev/null 2>&1 || true");
+                                $commands->push("docker network rm {$safe} >/dev/null 2>&1 || true");
                             }
                         }
                     }
@@ -210,7 +253,7 @@ class Init extends Command
                             'save_s3' => false,
                             'frequency' => '0 0 * * *',
                             'database_id' => $database->id,
-                            'database_type' => \App\Models\StandalonePostgresql::class,
+                            'database_type' => StandalonePostgresql::class,
                             'team_id' => 0,
                         ]);
                     }
@@ -225,13 +268,6 @@ class Init extends Command
     {
         $id = config('app.id');
         $version = config('constants.coolify.version');
-        $settings = instanceSettings();
-        $do_not_track = data_get($settings, 'do_not_track');
-        if ($do_not_track == true) {
-            echo "Do_not_track is enabled\n";
-
-            return;
-        }
         try {
             Http::get("https://undead.coolify.io/v4/alive?appId=$id&version=$version");
         } catch (\Throwable $e) {
@@ -239,23 +275,6 @@ class Init extends Command
         }
     }
 
-    private function cleanupInProgressApplicationDeployments()
-    {
-        // Cleanup any failed deployments
-        try {
-            if (isCloud()) {
-                return;
-            }
-            $queued_inprogress_deployments = ApplicationDeploymentQueue::whereIn('status', [ApplicationDeploymentStatus::IN_PROGRESS->value, ApplicationDeploymentStatus::QUEUED->value])->get();
-            foreach ($queued_inprogress_deployments as $deployment) {
-                $deployment->status = ApplicationDeploymentStatus::FAILED->value;
-                $deployment->save();
-            }
-        } catch (\Throwable $e) {
-            echo "Error: {$e->getMessage()}\n";
-        }
-    }
-
     private function replaceSlashInEnvironmentName()
     {
         if (version_compare('4.0.0-beta.298', config('constants.coolify.version'), '<=')) {

app/Console/Commands/NotifyDemo.php

@@ -56,7 +56,7 @@ class NotifyDemo extends Command
               php artisan app:demo-notify {channel}
             </p>
             <div class="my-1">
-                <div class="text-yellow-500"> Channels: </div>
+                <div class="text-warning-500"> Channels: </div>
                 <ul class="text-coolify">
                     <li>email</li>
                     <li>discord</li>

app/Console/Commands/ScheduledJobDiagnostics.php

@@ -0,0 +1,255 @@
+<?php
+
+namespace App\Console\Commands;
+
+use App\Models\DockerCleanupExecution;
+use App\Models\ScheduledDatabaseBackup;
+use App\Models\ScheduledTask;
+use App\Models\Server;
+use App\Models\Team;
+use Illuminate\Console\Command;
+use Illuminate\Support\Carbon;
+use Illuminate\Support\Facades\Cache;
+
+class ScheduledJobDiagnostics extends Command
+{
+    protected $signature = 'scheduled:diagnostics
+        {--type=all : Type to inspect: docker-cleanup, backups, tasks, server-jobs, all}
+        {--server= : Filter by server ID}';
+
+    protected $description = 'Inspect dedup cache state and scheduling decisions for all scheduled jobs';
+
+    public function handle(): int
+    {
+        $type = $this->option('type');
+        $serverFilter = $this->option('server');
+
+        $this->outputHeartbeat();
+
+        if (in_array($type, ['all', 'docker-cleanup'])) {
+            $this->inspectDockerCleanups($serverFilter);
+        }
+
+        if (in_array($type, ['all', 'backups'])) {
+            $this->inspectBackups();
+        }
+
+        if (in_array($type, ['all', 'tasks'])) {
+            $this->inspectTasks();
+        }
+
+        if (in_array($type, ['all', 'server-jobs'])) {
+            $this->inspectServerJobs($serverFilter);
+        }
+
+        return self::SUCCESS;
+    }
+
+    private function outputHeartbeat(): void
+    {
+        $heartbeat = Cache::get('scheduled-job-manager:heartbeat');
+        if ($heartbeat) {
+            $age = Carbon::parse($heartbeat)->diffForHumans();
+            $this->info("Scheduler heartbeat: {$heartbeat} ({$age})");
+        } else {
+            $this->error('Scheduler heartbeat: MISSING — ScheduledJobManager may not be running');
+        }
+        $this->newLine();
+    }
+
+    private function inspectDockerCleanups(?string $serverFilter): void
+    {
+        $this->info('=== Docker Cleanup Jobs ===');
+
+        $servers = $this->getServers($serverFilter);
+
+        $rows = [];
+        foreach ($servers as $server) {
+            $frequency = data_get($server->settings, 'docker_cleanup_frequency', '0 * * * *');
+            if (isset(VALID_CRON_STRINGS[$frequency])) {
+                $frequency = VALID_CRON_STRINGS[$frequency];
+            }
+
+            $dedupKey = "docker-cleanup:{$server->id}";
+            $cacheValue = Cache::get($dedupKey);
+            $timezone = data_get($server->settings, 'server_timezone', config('app.timezone'));
+
+            if (validate_timezone($timezone) === false) {
+                $timezone = config('app.timezone');
+            }
+
+            $wouldFire = shouldRunCronNow($frequency, $timezone, $dedupKey);
+
+            $lastExecution = DockerCleanupExecution::where('server_id', $server->id)
+                ->latest()
+                ->first();
+
+            $rows[] = [
+                $server->id,
+                $server->name,
+                $timezone,
+                $frequency,
+                $dedupKey,
+                $cacheValue ?? '<missing>',
+                $wouldFire ? 'YES' : 'no',
+                $lastExecution ? $lastExecution->status.' @ '.$lastExecution->created_at : 'never',
+            ];
+        }
+
+        $this->table(
+            ['ID', 'Server', 'TZ', 'Frequency', 'Dedup Key', 'Cache Value', 'Would Fire', 'Last Execution'],
+            $rows
+        );
+        $this->newLine();
+    }
+
+    private function inspectBackups(): void
+    {
+        $this->info('=== Scheduled Backups ===');
+
+        $backups = ScheduledDatabaseBackup::with(['database'])
+            ->where('enabled', true)
+            ->get();
+
+        $rows = [];
+        foreach ($backups as $backup) {
+            $server = $backup->server();
+            $frequency = $backup->frequency;
+            if (isset(VALID_CRON_STRINGS[$frequency])) {
+                $frequency = VALID_CRON_STRINGS[$frequency];
+            }
+
+            $dedupKey = "scheduled-backup:{$backup->id}";
+            $cacheValue = Cache::get($dedupKey);
+            $timezone = $server ? data_get($server->settings, 'server_timezone', config('app.timezone')) : config('app.timezone');
+
+            if (validate_timezone($timezone) === false) {
+                $timezone = config('app.timezone');
+            }
+
+            $wouldFire = shouldRunCronNow($frequency, $timezone, $dedupKey);
+
+            $rows[] = [
+                $backup->id,
+                $backup->database_type ?? 'unknown',
+                $server?->name ?? 'N/A',
+                $frequency,
+                $cacheValue ?? '<missing>',
+                $wouldFire ? 'YES' : 'no',
+            ];
+        }
+
+        $this->table(
+            ['Backup ID', 'DB Type', 'Server', 'Frequency', 'Cache Value', 'Would Fire'],
+            $rows
+        );
+        $this->newLine();
+    }
+
+    private function inspectTasks(): void
+    {
+        $this->info('=== Scheduled Tasks ===');
+
+        $tasks = ScheduledTask::with(['service', 'application'])
+            ->where('enabled', true)
+            ->get();
+
+        $rows = [];
+        foreach ($tasks as $task) {
+            $server = $task->server();
+            $frequency = $task->frequency;
+            if (isset(VALID_CRON_STRINGS[$frequency])) {
+                $frequency = VALID_CRON_STRINGS[$frequency];
+            }
+
+            $dedupKey = "scheduled-task:{$task->id}";
+            $cacheValue = Cache::get($dedupKey);
+            $timezone = $server ? data_get($server->settings, 'server_timezone', config('app.timezone')) : config('app.timezone');
+
+            if (validate_timezone($timezone) === false) {
+                $timezone = config('app.timezone');
+            }
+
+            $wouldFire = shouldRunCronNow($frequency, $timezone, $dedupKey);
+
+            $rows[] = [
+                $task->id,
+                $task->name,
+                $server?->name ?? 'N/A',
+                $frequency,
+                $cacheValue ?? '<missing>',
+                $wouldFire ? 'YES' : 'no',
+            ];
+        }
+
+        $this->table(
+            ['Task ID', 'Name', 'Server', 'Frequency', 'Cache Value', 'Would Fire'],
+            $rows
+        );
+        $this->newLine();
+    }
+
+    private function inspectServerJobs(?string $serverFilter): void
+    {
+        $this->info('=== Server Manager Jobs ===');
+
+        $servers = $this->getServers($serverFilter);
+
+        $rows = [];
+        foreach ($servers as $server) {
+            $timezone = data_get($server->settings, 'server_timezone', config('app.timezone'));
+            if (validate_timezone($timezone) === false) {
+                $timezone = config('app.timezone');
+            }
+
+            $dedupKeys = [
+                "sentinel-restart:{$server->id}" => '0 0 * * *',
+                "server-patch-check:{$server->id}" => '0 0 * * 0',
+                "server-check:{$server->id}" => isCloud() ? '*/5 * * * *' : '* * * * *',
+                "server-storage-check:{$server->id}" => data_get($server->settings, 'server_disk_usage_check_frequency', '0 23 * * *'),
+            ];
+
+            foreach ($dedupKeys as $dedupKey => $frequency) {
+                if (isset(VALID_CRON_STRINGS[$frequency])) {
+                    $frequency = VALID_CRON_STRINGS[$frequency];
+                }
+
+                $cacheValue = Cache::get($dedupKey);
+                $wouldFire = shouldRunCronNow($frequency, $timezone, $dedupKey);
+
+                $rows[] = [
+                    $server->id,
+                    $server->name,
+                    $dedupKey,
+                    $frequency,
+                    $cacheValue ?? '<missing>',
+                    $wouldFire ? 'YES' : 'no',
+                ];
+            }
+        }
+
+        $this->table(
+            ['Server ID', 'Server', 'Dedup Key', 'Frequency', 'Cache Value', 'Would Fire'],
+            $rows
+        );
+        $this->newLine();
+    }
+
+    private function getServers(?string $serverFilter): \Illuminate\Support\Collection
+    {
+        $query = Server::with('settings')->where('ip', '!=', '1.2.3.4');
+
+        if ($serverFilter) {
+            $query->where('id', $serverFilter);
+        }
+
+        if (isCloud()) {
+            $servers = $query->whereRelation('team.subscription', 'stripe_invoice_paid', true)->get();
+            $own = Team::find(0)?->servers()->with('settings')->get() ?? collect();
+
+            return $servers->merge($own);
+        }
+
+        return $query->get();
+    }
+}

app/Console/Commands/Scheduler.php

@@ -1,23 +0,0 @@
-<?php
-
-namespace App\Console\Commands;
-
-use Illuminate\Console\Command;
-
-class Scheduler extends Command
-{
-    protected $signature = 'start:scheduler';
-
-    protected $description = 'Start Scheduler';
-
-    public function handle()
-    {
-        if (config('constants.horizon.is_scheduler_enabled')) {
-            $this->info('Scheduler is enabled on this server.');
-            $this->call('schedule:work');
-            exit(0);
-        } else {
-            exit(0);
-        }
-    }
-}

app/Console/Commands/ServicesDelete.php

@@ -6,7 +6,14 @@ use App\Jobs\DeleteResourceJob;
 use App\Models\Application;
 use App\Models\Server;
 use App\Models\Service;
+use App\Models\StandaloneClickhouse;
+use App\Models\StandaloneDragonfly;
+use App\Models\StandaloneKeydb;
+use App\Models\StandaloneMariadb;
+use App\Models\StandaloneMongodb;
+use App\Models\StandaloneMysql;
 use App\Models\StandalonePostgresql;
+use App\Models\StandaloneRedis;
 use Illuminate\Console\Command;
 
 use function Laravel\Prompts\confirm;
@@ -103,19 +110,79 @@ class ServicesDelete extends Command
 
     private function deleteDatabase()
     {
-        $databases = StandalonePostgresql::all();
-        if ($databases->count() === 0) {
+        // Collect all databases from all types with unique identifiers
+        $allDatabases = collect();
+        $databaseOptions = collect();
+
+        // Add PostgreSQL databases
+        foreach (StandalonePostgresql::all() as $db) {
+            $key = "postgresql_{$db->id}";
+            $allDatabases->put($key, $db);
+            $databaseOptions->put($key, "{$db->name} (PostgreSQL)");
+        }
+
+        // Add MySQL databases
+        foreach (StandaloneMysql::all() as $db) {
+            $key = "mysql_{$db->id}";
+            $allDatabases->put($key, $db);
+            $databaseOptions->put($key, "{$db->name} (MySQL)");
+        }
+
+        // Add MariaDB databases
+        foreach (StandaloneMariadb::all() as $db) {
+            $key = "mariadb_{$db->id}";
+            $allDatabases->put($key, $db);
+            $databaseOptions->put($key, "{$db->name} (MariaDB)");
+        }
+
+        // Add MongoDB databases
+        foreach (StandaloneMongodb::all() as $db) {
+            $key = "mongodb_{$db->id}";
+            $allDatabases->put($key, $db);
+            $databaseOptions->put($key, "{$db->name} (MongoDB)");
+        }
+
+        // Add Redis databases
+        foreach (StandaloneRedis::all() as $db) {
+            $key = "redis_{$db->id}";
+            $allDatabases->put($key, $db);
+            $databaseOptions->put($key, "{$db->name} (Redis)");
+        }
+
+        // Add KeyDB databases
+        foreach (StandaloneKeydb::all() as $db) {
+            $key = "keydb_{$db->id}";
+            $allDatabases->put($key, $db);
+            $databaseOptions->put($key, "{$db->name} (KeyDB)");
+        }
+
+        // Add Dragonfly databases
+        foreach (StandaloneDragonfly::all() as $db) {
+            $key = "dragonfly_{$db->id}";
+            $allDatabases->put($key, $db);
+            $databaseOptions->put($key, "{$db->name} (Dragonfly)");
+        }
+
+        // Add ClickHouse databases
+        foreach (StandaloneClickhouse::all() as $db) {
+            $key = "clickhouse_{$db->id}";
+            $allDatabases->put($key, $db);
+            $databaseOptions->put($key, "{$db->name} (ClickHouse)");
+        }
+
+        if ($allDatabases->count() === 0) {
             $this->error('There are no databases to delete.');
 
             return;
         }
+
         $databasesToDelete = multiselect(
             'What database do you want to delete?',
-            $databases->pluck('name', 'id')->sortKeys(),
+            $databaseOptions->sortKeys(),
         );
 
-        foreach ($databasesToDelete as $database) {
-            $toDelete = $databases->where('id', $database)->first();
+        foreach ($databasesToDelete as $databaseKey) {
+            $toDelete = $allDatabases->get($databaseKey);
             if ($toDelete) {
                 $this->info($toDelete);
                 $confirmed = confirm('Are you sure you want to delete all selected resources?');

app/Console/Commands/SyncBunny.php

@@ -44,14 +44,16 @@ class SyncBunny extends Command
         $compose_file_prod = 'docker-compose.prod.yml';
         $install_script = 'install.sh';
         $upgrade_script = 'upgrade.sh';
+        $upgrade_postgres_script = 'upgrade-postgres.sh';
         $production_env = '.env.production';
-        $service_template = 'service-templates.json';
+        $service_template = config('constants.services.file_name');
         $versions = 'versions.json';
 
         $compose_file_location = "$parent_dir/$compose_file";
         $compose_file_prod_location = "$parent_dir/$compose_file_prod";
         $install_script_location = "$parent_dir/scripts/install.sh";
         $upgrade_script_location = "$parent_dir/scripts/upgrade.sh";
+        $upgrade_postgres_script_location = "$parent_dir/scripts/upgrade-postgres.sh";
         $production_env_location = "$parent_dir/.env.production";
         $versions_location = "$parent_dir/$versions";
 
@@ -87,22 +89,86 @@ class SyncBunny extends Command
                 $compose_file_prod_location = "$parent_dir/other/nightly/$compose_file_prod";
                 $production_env_location = "$parent_dir/other/nightly/$production_env";
                 $upgrade_script_location = "$parent_dir/other/nightly/$upgrade_script";
+                $upgrade_postgres_script_location = "$parent_dir/other/nightly/$upgrade_postgres_script";
                 $install_script_location = "$parent_dir/other/nightly/$install_script";
                 $versions_location = "$parent_dir/other/nightly/$versions";
             }
             if (! $only_template && ! $only_version) {
-                if ($nightly) {
-                    $this->info('About to sync files NIGHTLY (docker-compose.prod.yaml, upgrade.sh, install.sh, etc) to BunnyCDN.');
-                } else {
-                    $this->info('About to sync files PRODUCTION (docker-compose.yml, docker-compose.prod.yml, upgrade.sh, install.sh, etc) to BunnyCDN.');
+                $envLabel = $nightly ? 'NIGHTLY' : 'PRODUCTION';
+                $this->info("About to sync $envLabel files to BunnyCDN.");
+                $this->newLine();
+
+                // BunnyCDN file mapping (local file => CDN URL path)
+                $bunnyFileMapping = [
+                    $compose_file_location => "$bunny_cdn/$bunny_cdn_path/$compose_file",
+                    $compose_file_prod_location => "$bunny_cdn/$bunny_cdn_path/$compose_file_prod",
+                    $production_env_location => "$bunny_cdn/$bunny_cdn_path/$production_env",
+                    $upgrade_script_location => "$bunny_cdn/$bunny_cdn_path/$upgrade_script",
+                    $upgrade_postgres_script_location => "$bunny_cdn/$bunny_cdn_path/$upgrade_postgres_script",
+                    $install_script_location => "$bunny_cdn/$bunny_cdn_path/$install_script",
+                ];
+
+                $diffTmpDir = sys_get_temp_dir().'/coolify-cdn-diff-'.time();
+                @mkdir($diffTmpDir, 0755, true);
+                $hasChanges = false;
+
+                // Diff against BunnyCDN
+                $this->info('Fetching files from BunnyCDN to compare...');
+                foreach ($bunnyFileMapping as $localFile => $cdnUrl) {
+                    if (! file_exists($localFile)) {
+                        $this->warn('Local file not found: '.$localFile);
+
+                        continue;
+                    }
+
+                    $fileName = basename($cdnUrl);
+                    $remoteTmp = "$diffTmpDir/bunny-$fileName";
+
+                    try {
+                        $response = Http::timeout(10)->get($cdnUrl);
+                        if ($response->successful()) {
+                            file_put_contents($remoteTmp, $response->body());
+                            $diffOutput = [];
+                            exec('diff -u '.escapeshellarg($remoteTmp).' '.escapeshellarg($localFile).' 2>&1', $diffOutput, $diffCode);
+                            if ($diffCode !== 0) {
+                                $hasChanges = true;
+                                $this->newLine();
+                                $this->info("--- BunnyCDN: $bunny_cdn_path/$fileName");
+                                $this->info("+++ Local: $fileName");
+                                foreach ($diffOutput as $line) {
+                                    if (str_starts_with($line, '---') || str_starts_with($line, '+++')) {
+                                        continue;
+                                    }
+                                    $this->line($line);
+                                }
+                            }
+                        } else {
+                            $this->info("NEW on BunnyCDN: $bunny_cdn_path/$fileName (HTTP {$response->status()})");
+                            $hasChanges = true;
+                        }
+                    } catch (\Throwable $e) {
+                        $this->warn("Could not fetch $cdnUrl: {$e->getMessage()}");
+                    }
                 }
+
+                exec('rm -rf '.escapeshellarg($diffTmpDir));
+
+                if (! $hasChanges) {
+                    $this->newLine();
+                    $this->info('No differences found. All files are already up to date.');
+
+                    return;
+                }
+
+                $this->newLine();
+
                 $confirmed = confirm('Are you sure you want to sync?');
                 if (! $confirmed) {
                     return;
                 }
             }
             if ($only_template) {
-                $this->info('About to sync service-templates.json to BunnyCDN.');
+                $this->info('About to sync '.config('constants.services.file_name').' to BunnyCDN.');
                 $confirmed = confirm('Are you sure you want to sync?');
                 if (! $confirmed) {
                     return;
@@ -116,7 +182,7 @@ class SyncBunny extends Command
                 return;
             } elseif ($only_version) {
                 if ($nightly) {
-                    $this->info('About to sync NIGHLTY versions.json to BunnyCDN.');
+                    $this->info('About to sync NIGHTLY versions.json to BunnyCDN.');
                 } else {
                     $this->info('About to sync PRODUCTION versions.json to BunnyCDN.');
                 }
@@ -124,15 +190,26 @@ class SyncBunny extends Command
                 $json = json_decode($file, true);
                 $actual_version = data_get($json, 'coolify.v4.version');
 
-                $confirmed = confirm("Are you sure you want to sync to {$actual_version}?");
+                $this->info("Version: {$actual_version}");
+                $this->info('This will:');
+                $this->info('  1. Sync versions.json to BunnyCDN');
+                $this->newLine();
+
+                $confirmed = confirm('Are you sure you want to proceed?');
                 if (! $confirmed) {
                     return;
                 }
+
+                $this->info('Syncing versions.json to BunnyCDN...');
                 Http::pool(fn (Pool $pool) => [
                     $pool->storage(fileName: $versions_location)->put("/$bunny_cdn_storage_name/$bunny_cdn_path/$versions"),
                     $pool->purge("$bunny_cdn/$bunny_cdn_path/$versions"),
                 ]);
-                $this->info('versions.json uploaded & purged...');
+                $this->info('✓ versions.json uploaded & purged to BunnyCDN');
+                $this->newLine();
+
+                $this->info('=== Summary ===');
+                $this->info('BunnyCDN sync: ✓ Complete');
 
                 return;
             }
@@ -142,6 +219,7 @@ class SyncBunny extends Command
                 $pool->storage(fileName: "$compose_file_prod_location")->put("/$bunny_cdn_storage_name/$bunny_cdn_path/$compose_file_prod"),
                 $pool->storage(fileName: "$production_env_location")->put("/$bunny_cdn_storage_name/$bunny_cdn_path/$production_env"),
                 $pool->storage(fileName: "$upgrade_script_location")->put("/$bunny_cdn_storage_name/$bunny_cdn_path/$upgrade_script"),
+                $pool->storage(fileName: "$upgrade_postgres_script_location")->put("/$bunny_cdn_storage_name/$bunny_cdn_path/$upgrade_postgres_script"),
                 $pool->storage(fileName: "$install_script_location")->put("/$bunny_cdn_storage_name/$bunny_cdn_path/$install_script"),
             ]);
             Http::pool(fn (Pool $pool) => [
@@ -149,9 +227,14 @@ class SyncBunny extends Command
                 $pool->purge("$bunny_cdn/$bunny_cdn_path/$compose_file_prod"),
                 $pool->purge("$bunny_cdn/$bunny_cdn_path/$production_env"),
                 $pool->purge("$bunny_cdn/$bunny_cdn_path/$upgrade_script"),
+                $pool->purge("$bunny_cdn/$bunny_cdn_path/$upgrade_postgres_script"),
                 $pool->purge("$bunny_cdn/$bunny_cdn_path/$install_script"),
             ]);
-            $this->info('All files uploaded & purged...');
+            $this->info('All files uploaded & purged to BunnyCDN.');
+            $this->newLine();
+
+            $this->info('=== Summary ===');
+            $this->info('BunnyCDN sync: Complete');
         } catch (\Throwable $e) {
             $this->error('Error: '.$e->getMessage());
         }

app/Console/Commands/UpdateServiceVersions.php

@@ -0,0 +1,791 @@
+<?php
+
+namespace App\Console\Commands;
+
+use Illuminate\Console\Command;
+use Illuminate\Support\Facades\Http;
+use Symfony\Component\Yaml\Yaml;
+
+class UpdateServiceVersions extends Command
+{
+    protected $signature = 'services:update-versions
+                            {--service= : Update specific service template}
+                            {--dry-run : Show what would be updated without making changes}
+                            {--registry= : Filter by registry (dockerhub, ghcr, quay, codeberg)}';
+
+    protected $description = 'Update service template files with latest Docker image versions from registries';
+
+    protected array $stats = [
+        'total' => 0,
+        'updated' => 0,
+        'failed' => 0,
+        'skipped' => 0,
+    ];
+
+    protected array $registryCache = [];
+
+    protected array $majorVersionUpdates = [];
+
+    public function handle(): int
+    {
+        $this->info('Starting service version update...');
+
+        $templateFiles = $this->getTemplateFiles();
+
+        $this->stats['total'] = count($templateFiles);
+
+        foreach ($templateFiles as $file) {
+            $this->processTemplate($file);
+        }
+
+        $this->newLine();
+        $this->displayStats();
+
+        return self::SUCCESS;
+    }
+
+    protected function getTemplateFiles(): array
+    {
+        $pattern = base_path('templates/compose/*.yaml');
+        $files = glob($pattern);
+
+        if ($service = $this->option('service')) {
+            $files = array_filter($files, fn ($file) => basename($file) === "$service.yaml");
+        }
+
+        return $files;
+    }
+
+    protected function processTemplate(string $filePath): void
+    {
+        $filename = basename($filePath);
+        $this->info("Processing: {$filename}");
+
+        try {
+            $content = file_get_contents($filePath);
+            $yaml = Yaml::parse($content);
+
+            if (! isset($yaml['services'])) {
+                $this->warn("  No services found in {$filename}");
+                $this->stats['skipped']++;
+
+                return;
+            }
+
+            $updated = false;
+            $updatedYaml = $yaml;
+
+            foreach ($yaml['services'] as $serviceName => $serviceConfig) {
+                if (! isset($serviceConfig['image'])) {
+                    continue;
+                }
+
+                $currentImage = $serviceConfig['image'];
+
+                // Check if using 'latest' tag and log for manual review
+                if (str_contains($currentImage, ':latest')) {
+                    $registryUrl = $this->getRegistryUrl($currentImage);
+                    $this->warn("  {$serviceName}: {$currentImage} (using 'latest' tag)");
+                    if ($registryUrl) {
+                        $this->line("    → Manual review: {$registryUrl}");
+                    }
+                }
+
+                $latestVersion = $this->getLatestVersion($currentImage);
+
+                if ($latestVersion && $latestVersion !== $currentImage) {
+                    $this->line("  {$serviceName}: {$currentImage} → {$latestVersion}");
+                    $updatedYaml['services'][$serviceName]['image'] = $latestVersion;
+                    $updated = true;
+                } else {
+                    $this->line("  {$serviceName}: {$currentImage} (up to date)");
+                }
+            }
+
+            if ($updated) {
+                if (! $this->option('dry-run')) {
+                    $this->updateYamlFile($filePath, $content, $updatedYaml);
+                    $this->stats['updated']++;
+                } else {
+                    $this->warn('  [DRY RUN] Would update this file');
+                    $this->stats['updated']++;
+                }
+            } else {
+                $this->stats['skipped']++;
+            }
+
+        } catch (\Throwable $e) {
+            $this->error("  Failed: {$e->getMessage()}");
+            $this->stats['failed']++;
+        }
+
+        $this->newLine();
+    }
+
+    protected function getLatestVersion(string $image): ?string
+    {
+        // Parse the image string
+        [$repository, $currentTag] = $this->parseImage($image);
+
+        // Determine registry and fetch latest version
+        $result = null;
+        if (str_starts_with($repository, 'ghcr.io/')) {
+            $result = $this->getGhcrLatestVersion($repository, $currentTag);
+        } elseif (str_starts_with($repository, 'quay.io/')) {
+            $result = $this->getQuayLatestVersion($repository, $currentTag);
+        } elseif (str_starts_with($repository, 'codeberg.org/')) {
+            $result = $this->getCodebergLatestVersion($repository, $currentTag);
+        } elseif (str_starts_with($repository, 'lscr.io/')) {
+            $result = $this->getDockerHubLatestVersion($repository, $currentTag);
+        } elseif ($this->isCustomRegistry($repository)) {
+            // Custom registries - skip for now, log warning
+            $this->warn("  Skipping custom registry: {$repository}");
+            $result = null;
+        } else {
+            // DockerHub (default registry - no prefix or docker.io/index.docker.io)
+            $result = $this->getDockerHubLatestVersion($repository, $currentTag);
+        }
+
+        return $result;
+    }
+
+    protected function isCustomRegistry(string $repository): bool
+    {
+        // List of custom/private registries that we can't query
+        $customRegistries = [
+            'docker.elastic.co/',
+            'docker.n8n.io/',
+            'docker.flipt.io/',
+            'docker.getoutline.com/',
+            'cr.weaviate.io/',
+            'downloads.unstructured.io/',
+            'budibase.docker.scarf.sh/',
+            'calcom.docker.scarf.sh/',
+            'code.forgejo.org/',
+            'registry.supertokens.io/',
+            'registry.rocket.chat/',
+            'nabo.codimd.dev/',
+            'gcr.io/',
+        ];
+
+        foreach ($customRegistries as $registry) {
+            if (str_starts_with($repository, $registry)) {
+                return true;
+            }
+        }
+
+        return false;
+    }
+
+    protected function getRegistryUrl(string $image): ?string
+    {
+        [$repository] = $this->parseImage($image);
+
+        // GitHub Container Registry
+        if (str_starts_with($repository, 'ghcr.io/')) {
+            $parts = explode('/', str_replace('ghcr.io/', '', $repository));
+            if (count($parts) >= 2) {
+                return "https://github.com/{$parts[0]}/{$parts[1]}/pkgs/container/{$parts[1]}";
+            }
+        }
+
+        // Quay.io
+        if (str_starts_with($repository, 'quay.io/')) {
+            $repo = str_replace('quay.io/', '', $repository);
+
+            return "https://quay.io/repository/{$repo}?tab=tags";
+        }
+
+        // Codeberg
+        if (str_starts_with($repository, 'codeberg.org/')) {
+            $parts = explode('/', str_replace('codeberg.org/', '', $repository));
+            if (count($parts) >= 2) {
+                return "https://codeberg.org/{$parts[0]}/-/packages/container/{$parts[1]}";
+            }
+        }
+
+        // Docker Hub
+        $cleanRepo = str_replace(['index.docker.io/', 'docker.io/', 'lscr.io/'], '', $repository);
+        if (! str_contains($cleanRepo, '/')) {
+            // Official image
+            return "https://hub.docker.com/_/{$cleanRepo}/tags";
+        } else {
+            // User/org image
+            return "https://hub.docker.com/r/{$cleanRepo}/tags";
+        }
+    }
+
+    protected function parseImage(string $image): array
+    {
+        if (str_contains($image, ':')) {
+            [$repo, $tag] = explode(':', $image, 2);
+        } else {
+            $repo = $image;
+            $tag = 'latest';
+        }
+
+        // Handle variables in tags
+        if (str_contains($tag, '$')) {
+            $tag = 'latest'; // Default to latest for variable tags
+        }
+
+        return [$repo, $tag];
+    }
+
+    protected function getDockerHubLatestVersion(string $repository, string $currentTag): ?string
+    {
+        try {
+            // Check if we've already fetched tags for this repository
+            if (! isset($this->registryCache[$repository.'_tags'])) {
+                // Remove various registry prefixes
+                $cleanRepo = $repository;
+                $cleanRepo = str_replace('index.docker.io/', '', $cleanRepo);
+                $cleanRepo = str_replace('docker.io/', '', $cleanRepo);
+                $cleanRepo = str_replace('lscr.io/', '', $cleanRepo);
+
+                // For official images (no /) add library prefix
+                if (! str_contains($cleanRepo, '/')) {
+                    $cleanRepo = "library/{$cleanRepo}";
+                }
+
+                $url = "https://hub.docker.com/v2/repositories/{$cleanRepo}/tags";
+
+                $response = Http::timeout(10)->get($url, [
+                    'page_size' => 100,
+                    'ordering' => 'last_updated',
+                ]);
+
+                if (! $response->successful()) {
+                    return null;
+                }
+
+                $data = $response->json();
+                $tags = $data['results'] ?? [];
+
+                // Cache the tags for this repository
+                $this->registryCache[$repository.'_tags'] = $tags;
+            } else {
+                $this->line("    [cached] Using cached tags for {$repository}");
+                $tags = $this->registryCache[$repository.'_tags'];
+            }
+
+            // Find the best matching tag
+            return $this->findBestTag($tags, $currentTag, $repository);
+
+        } catch (\Throwable $e) {
+            $this->warn("  DockerHub API error for {$repository}: {$e->getMessage()}");
+
+            return null;
+        }
+    }
+
+    protected function findLatestTagDigest(array $tags, string $targetTag = 'latest'): ?string
+    {
+        // Find the digest/sha for the target tag (usually 'latest')
+        foreach ($tags as $tag) {
+            if ($tag['name'] === $targetTag) {
+                return $tag['digest'] ?? $tag['images'][0]['digest'] ?? null;
+            }
+        }
+
+        return null;
+    }
+
+    protected function findVersionTagsForDigest(array $tags, string $digest): array
+    {
+        // Find all semantic version tags that share the same digest
+        $versionTags = [];
+
+        foreach ($tags as $tag) {
+            $tagDigest = $tag['digest'] ?? $tag['images'][0]['digest'] ?? null;
+
+            if ($tagDigest === $digest) {
+                $tagName = $tag['name'];
+                // Only include semantic version tags
+                if (preg_match('/^\d+\.\d+(\.\d+)?$/', $tagName)) {
+                    $versionTags[] = $tagName;
+                }
+            }
+        }
+
+        return $versionTags;
+    }
+
+    protected function getGhcrLatestVersion(string $repository, string $currentTag): ?string
+    {
+        try {
+            // GHCR doesn't have a public API for listing tags without auth
+            // We'll try to fetch the package metadata via GitHub API
+            $parts = explode('/', str_replace('ghcr.io/', '', $repository));
+
+            if (count($parts) < 2) {
+                return null;
+            }
+
+            $owner = $parts[0];
+            $package = $parts[1];
+
+            // Try GitHub Container Registry API
+            $url = "https://api.github.com/users/{$owner}/packages/container/{$package}/versions";
+
+            $response = Http::timeout(10)
+                ->withHeaders([
+                    'Accept' => 'application/vnd.github.v3+json',
+                ])
+                ->get($url, ['per_page' => 100]);
+
+            if (! $response->successful()) {
+                // Most GHCR packages require authentication
+                if ($currentTag === 'latest') {
+                    $this->warn('    ⚠ GHCR requires authentication - manual review needed');
+                }
+
+                return null;
+            }
+
+            $versions = $response->json();
+            $tags = [];
+
+            // Build tags array with digest information
+            foreach ($versions as $version) {
+                $digest = $version['name'] ?? null; // This is the SHA digest
+
+                if (isset($version['metadata']['container']['tags'])) {
+                    foreach ($version['metadata']['container']['tags'] as $tag) {
+                        $tags[] = [
+                            'name' => $tag,
+                            'digest' => $digest,
+                        ];
+                    }
+                }
+            }
+
+            return $this->findBestTag($tags, $currentTag, $repository);
+
+        } catch (\Throwable $e) {
+            $this->warn("  GHCR API error for {$repository}: {$e->getMessage()}");
+
+            return null;
+        }
+    }
+
+    protected function getQuayLatestVersion(string $repository, string $currentTag): ?string
+    {
+        try {
+            // Check if we've already fetched tags for this repository
+            if (! isset($this->registryCache[$repository.'_tags'])) {
+                $cleanRepo = str_replace('quay.io/', '', $repository);
+
+                $url = "https://quay.io/api/v1/repository/{$cleanRepo}/tag/";
+
+                $response = Http::timeout(10)->get($url, ['limit' => 100]);
+
+                if (! $response->successful()) {
+                    return null;
+                }
+
+                $data = $response->json();
+                $tags = array_map(fn ($tag) => ['name' => $tag['name']], $data['tags'] ?? []);
+
+                // Cache the tags for this repository
+                $this->registryCache[$repository.'_tags'] = $tags;
+            } else {
+                $this->line("    [cached] Using cached tags for {$repository}");
+                $tags = $this->registryCache[$repository.'_tags'];
+            }
+
+            return $this->findBestTag($tags, $currentTag, $repository);
+
+        } catch (\Throwable $e) {
+            $this->warn("  Quay API error for {$repository}: {$e->getMessage()}");
+
+            return null;
+        }
+    }
+
+    protected function getCodebergLatestVersion(string $repository, string $currentTag): ?string
+    {
+        try {
+            // Check if we've already fetched tags for this repository
+            if (! isset($this->registryCache[$repository.'_tags'])) {
+                // Codeberg uses Forgejo/Gitea, which has a container registry API
+                $cleanRepo = str_replace('codeberg.org/', '', $repository);
+                $parts = explode('/', $cleanRepo);
+
+                if (count($parts) < 2) {
+                    return null;
+                }
+
+                $owner = $parts[0];
+                $package = $parts[1];
+
+                // Codeberg API endpoint for packages
+                $url = "https://codeberg.org/api/packages/{$owner}/container/{$package}";
+
+                $response = Http::timeout(10)->get($url);
+
+                if (! $response->successful()) {
+                    return null;
+                }
+
+                $data = $response->json();
+                $tags = [];
+
+                if (isset($data['versions'])) {
+                    foreach ($data['versions'] as $version) {
+                        if (isset($version['name'])) {
+                            $tags[] = ['name' => $version['name']];
+                        }
+                    }
+                }
+
+                // Cache the tags for this repository
+                $this->registryCache[$repository.'_tags'] = $tags;
+            } else {
+                $this->line("    [cached] Using cached tags for {$repository}");
+                $tags = $this->registryCache[$repository.'_tags'];
+            }
+
+            return $this->findBestTag($tags, $currentTag, $repository);
+
+        } catch (\Throwable $e) {
+            $this->warn("  Codeberg API error for {$repository}: {$e->getMessage()}");
+
+            return null;
+        }
+    }
+
+    protected function findBestTag(array $tags, string $currentTag, string $repository): ?string
+    {
+        if (empty($tags)) {
+            return null;
+        }
+
+        // If current tag is 'latest', find what version it actually points to
+        if ($currentTag === 'latest') {
+            // First, try to find the digest for 'latest' tag
+            $latestDigest = $this->findLatestTagDigest($tags, 'latest');
+
+            if ($latestDigest) {
+                // Find all semantic version tags that share the same digest
+                $versionTags = $this->findVersionTagsForDigest($tags, $latestDigest);
+
+                if (! empty($versionTags)) {
+                    // Prefer shorter version tags (1.8 over 1.8.1)
+                    $bestVersion = $this->preferShorterVersion($versionTags);
+                    $this->info("    ✓ Found 'latest' points to: {$bestVersion}");
+
+                    return $repository.':'.$bestVersion;
+                }
+            }
+
+            // Fallback: get the latest semantic version available (prefer shorter)
+            $semverTags = $this->filterSemanticVersionTags($tags);
+            if (! empty($semverTags)) {
+                $bestVersion = $this->preferShorterVersion($semverTags);
+
+                return $repository.':'.$bestVersion;
+            }
+
+            // If no semantic versions found, keep 'latest'
+            return null;
+        }
+
+        // Check for major version updates for reporting
+        $this->checkForMajorVersionUpdate($tags, $currentTag, $repository);
+
+        // If current tag is a major version (e.g., "8", "5", "16")
+        if (preg_match('/^\d+$/', $currentTag)) {
+            $majorVersion = (int) $currentTag;
+            $matchingTags = array_filter($tags, function ($tag) use ($majorVersion) {
+                $name = $tag['name'];
+
+                // Match tags that start with the major version
+                return preg_match("/^{$majorVersion}(\.\d+)?(\.\d+)?$/", $name);
+            });
+
+            if (! empty($matchingTags)) {
+                $versions = array_column($matchingTags, 'name');
+                $bestVersion = $this->preferShorterVersion($versions);
+                if ($bestVersion !== $currentTag) {
+                    return $repository.':'.$bestVersion;
+                }
+            }
+        }
+
+        // If current tag is date-based version (e.g., "2025.06.02-sha-xxx")
+        if (preg_match('/^\d{4}\.\d{2}\.\d{2}/', $currentTag)) {
+            // Get all date-based tags
+            $dateTags = array_filter($tags, function ($tag) {
+                return preg_match('/^\d{4}\.\d{2}\.\d{2}/', $tag['name']);
+            });
+
+            if (! empty($dateTags)) {
+                $versions = array_column($dateTags, 'name');
+                $sorted = $this->sortSemanticVersions($versions);
+                $latestDate = $sorted[0];
+
+                // Compare dates
+                if ($latestDate !== $currentTag) {
+                    return $repository.':'.$latestDate;
+                }
+            }
+
+            return null;
+        }
+
+        // If current tag is semantic version (e.g., "1.7.4", "8.0")
+        if (preg_match('/^\d+\.\d+(\.\d+)?$/', $currentTag)) {
+            $parts = explode('.', $currentTag);
+            $majorMinor = $parts[0].'.'.$parts[1];
+
+            $matchingTags = array_filter($tags, function ($tag) use ($majorMinor) {
+                $name = $tag['name'];
+
+                return str_starts_with($name, $majorMinor);
+            });
+
+            if (! empty($matchingTags)) {
+                $versions = array_column($matchingTags, 'name');
+                $bestVersion = $this->preferShorterVersion($versions);
+                if (version_compare($bestVersion, $currentTag, '>') || version_compare($bestVersion, $currentTag, '=')) {
+                    // Only update if it's newer or if we can simplify (1.8.1 -> 1.8)
+                    if ($bestVersion !== $currentTag) {
+                        return $repository.':'.$bestVersion;
+                    }
+                }
+            }
+        }
+
+        // If current tag is a named version (e.g., "stable")
+        if (in_array($currentTag, ['stable', 'lts', 'edge'])) {
+            // Check if the same tag exists in the list (it's up to date)
+            $exists = array_filter($tags, fn ($tag) => $tag['name'] === $currentTag);
+            if (! empty($exists)) {
+                return null; // Tag exists and is current
+            }
+        }
+
+        return null;
+    }
+
+    protected function filterSemanticVersionTags(array $tags): array
+    {
+        $semverTags = array_filter($tags, function ($tag) {
+            $name = $tag['name'];
+
+            // Accept semantic versions (1.2.3, v1.2.3)
+            if (preg_match('/^v?\d+\.\d+(\.\d+)?(\.\d+)?$/', $name)) {
+                // Exclude versions with suffixes like -rc, -beta, -alpha
+                if (preg_match('/-(rc|beta|alpha|dev|test|pre|snapshot)/i', $name)) {
+                    return false;
+                }
+
+                return true;
+            }
+
+            // Accept date-based versions (2025.06.02, 2025.10.0, 2025.06.02-sha-xxx, RELEASE.2025-10-15T17-29-55Z)
+            if (preg_match('/^\d{4}\.\d{2}\.(\d{2}|\d)/', $name) || preg_match('/^RELEASE\.\d{4}-\d{2}-\d{2}/', $name)) {
+                return true;
+            }
+
+            return false;
+        });
+
+        return $this->sortSemanticVersions(array_column($semverTags, 'name'));
+    }
+
+    protected function sortSemanticVersions(array $versions): array
+    {
+        usort($versions, function ($a, $b) {
+            // Check if these are date-based versions (YYYY.MM.DD or YYYY.MM.D format)
+            $isDateA = preg_match('/^(\d{4})\.(\d{2})\.(\d{1,2})/', $a, $matchesA);
+            $isDateB = preg_match('/^(\d{4})\.(\d{2})\.(\d{1,2})/', $b, $matchesB);
+
+            if ($isDateA && $isDateB) {
+                // Both are date-based (YYYY.MM.DD), compare as dates
+                $dateA = $matchesA[1].$matchesA[2].str_pad($matchesA[3], 2, '0', STR_PAD_LEFT); // YYYYMMDD
+                $dateB = $matchesB[1].$matchesB[2].str_pad($matchesB[3], 2, '0', STR_PAD_LEFT); // YYYYMMDD
+
+                return strcmp($dateB, $dateA); // Descending order (newest first)
+            }
+
+            // Check if these are RELEASE date versions (RELEASE.YYYY-MM-DDTHH-MM-SSZ)
+            $isReleaseA = preg_match('/^RELEASE\.(\d{4})-(\d{2})-(\d{2})T(\d{2})-(\d{2})-(\d{2})Z/', $a, $matchesA);
+            $isReleaseB = preg_match('/^RELEASE\.(\d{4})-(\d{2})-(\d{2})T(\d{2})-(\d{2})-(\d{2})Z/', $b, $matchesB);
+
+            if ($isReleaseA && $isReleaseB) {
+                // Both are RELEASE format, compare as datetime
+                $dateTimeA = $matchesA[1].$matchesA[2].$matchesA[3].$matchesA[4].$matchesA[5].$matchesA[6]; // YYYYMMDDHHMMSS
+                $dateTimeB = $matchesB[1].$matchesB[2].$matchesB[3].$matchesB[4].$matchesB[5].$matchesB[6]; // YYYYMMDDHHMMSS
+
+                return strcmp($dateTimeB, $dateTimeA); // Descending order (newest first)
+            }
+
+            // Strip 'v' prefix for version comparison
+            $cleanA = ltrim($a, 'v');
+            $cleanB = ltrim($b, 'v');
+
+            // Fall back to semantic version comparison
+            return version_compare($cleanB, $cleanA); // Descending order
+        });
+
+        return $versions;
+    }
+
+    protected function preferShorterVersion(array $versions): string
+    {
+        if (empty($versions)) {
+            return '';
+        }
+
+        // Sort by version (highest first)
+        $sorted = $this->sortSemanticVersions($versions);
+        $highest = $sorted[0];
+
+        // Parse the highest version
+        $parts = explode('.', $highest);
+
+        // Look for shorter versions that match
+        // Priority: major (8) > major.minor (8.0) > major.minor.patch (8.0.39)
+
+        // Try to find just major.minor (e.g., 1.8 instead of 1.8.1)
+        if (count($parts) === 3) {
+            $majorMinor = $parts[0].'.'.$parts[1];
+            if (in_array($majorMinor, $versions)) {
+                return $majorMinor;
+            }
+        }
+
+        // Try to find just major (e.g., 8 instead of 8.0.39)
+        if (count($parts) >= 2) {
+            $major = $parts[0];
+            if (in_array($major, $versions)) {
+                return $major;
+            }
+        }
+
+        // Return the highest version we found
+        return $highest;
+    }
+
+    protected function updateYamlFile(string $filePath, string $originalContent, array $updatedYaml): void
+    {
+        // Preserve comments and formatting by updating the YAML content
+        $lines = explode("\n", $originalContent);
+        $updatedLines = [];
+        $inServices = false;
+        $currentService = null;
+
+        foreach ($lines as $line) {
+            // Detect if we're in the services section
+            if (preg_match('/^services:/', $line)) {
+                $inServices = true;
+                $updatedLines[] = $line;
+
+                continue;
+            }
+
+            // Detect service name (allow hyphens and underscores)
+            if ($inServices && preg_match('/^  ([\w-]+):/', $line, $matches)) {
+                $currentService = $matches[1];
+                $updatedLines[] = $line;
+
+                continue;
+            }
+
+            // Update image line
+            if ($currentService && preg_match('/^(\s+)image:\s*(.+)$/', $line, $matches)) {
+                $indent = $matches[1];
+                $newImage = $updatedYaml['services'][$currentService]['image'] ?? $matches[2];
+                $updatedLines[] = "{$indent}image: {$newImage}";
+
+                continue;
+            }
+
+            // If we hit a non-indented line, we're out of services
+            if ($inServices && preg_match('/^\S/', $line) && ! preg_match('/^services:/', $line)) {
+                $inServices = false;
+                $currentService = null;
+            }
+
+            $updatedLines[] = $line;
+        }
+
+        file_put_contents($filePath, implode("\n", $updatedLines));
+    }
+
+    protected function checkForMajorVersionUpdate(array $tags, string $currentTag, string $repository): void
+    {
+        // Only check semantic versions
+        if (! preg_match('/^v?(\d+)\./', $currentTag, $currentMatches)) {
+            return;
+        }
+
+        $currentMajor = (int) $currentMatches[1];
+
+        // Get all semantic version tags
+        $semverTags = $this->filterSemanticVersionTags($tags);
+
+        // Find the highest major version available
+        $highestMajor = $currentMajor;
+        foreach ($semverTags as $version) {
+            if (preg_match('/^v?(\d+)\./', $version, $matches)) {
+                $major = (int) $matches[1];
+                if ($major > $highestMajor) {
+                    $highestMajor = $major;
+                }
+            }
+        }
+
+        // If there's a higher major version available, record it
+        if ($highestMajor > $currentMajor) {
+            $this->majorVersionUpdates[] = [
+                'repository' => $repository,
+                'current' => $currentTag,
+                'current_major' => $currentMajor,
+                'available_major' => $highestMajor,
+                'registry_url' => $this->getRegistryUrl($repository.':'.$currentTag),
+            ];
+        }
+    }
+
+    protected function displayStats(): void
+    {
+        $this->info('Summary:');
+        $this->table(
+            ['Metric', 'Count'],
+            [
+                ['Total Templates', $this->stats['total']],
+                ['Updated', $this->stats['updated']],
+                ['Skipped (up to date)', $this->stats['skipped']],
+                ['Failed', $this->stats['failed']],
+            ]
+        );
+
+        // Display major version updates if any
+        if (! empty($this->majorVersionUpdates)) {
+            $this->newLine();
+            $this->warn('⚠ Services with available MAJOR version updates:');
+            $this->newLine();
+
+            $tableData = [];
+            foreach ($this->majorVersionUpdates as $update) {
+                $tableData[] = [
+                    $update['repository'],
+                    "v{$update['current_major']}.x",
+                    "v{$update['available_major']}.x",
+                    $update['registry_url'],
+                ];
+            }
+
+            $this->table(
+                ['Repository', 'Current', 'Available', 'Registry URL'],
+                $tableData
+            );
+
+            $this->newLine();
+            $this->comment('💡 Major version updates may include breaking changes. Review before upgrading.');
+        }
+    }
+}

app/Console/Commands/ViewScheduledLogs.php

@@ -28,6 +28,11 @@ class ViewScheduledLogs extends Command
     public function handle()
     {
         $date = $this->option('date') ?: now()->format('Y-m-d');
+        if (! preg_match('/^\d{4}-\d{2}-\d{2}$/', $date)) {
+            $this->error('Invalid date format. Use Y-m-d (e.g. 2025-01-31).');
+
+            return self::INVALID;
+        }
         $logPaths = $this->getLogPaths($date);
 
         if (empty($logPaths)) {
@@ -49,17 +54,19 @@ class ViewScheduledLogs extends Command
             $this->line('');
 
             if (count($logPaths) === 1) {
-                $logPath = $logPaths[0];
+                $logPath = escapeshellarg($logPaths[0]);
                 if ($filters) {
-                    passthru("tail -f {$logPath} | grep -E '{$filters}'");
+                    $escapedFilters = escapeshellarg($filters);
+                    passthru("tail -f {$logPath} | grep -E {$escapedFilters}");
                 } else {
                     passthru("tail -f {$logPath}");
                 }
             } else {
                 // Multiple files - use multitail or tail with process substitution
-                $logPathsStr = implode(' ', $logPaths);
+                $logPathsStr = implode(' ', array_map('escapeshellarg', $logPaths));
                 if ($filters) {
-                    passthru("tail -f {$logPathsStr} | grep -E '{$filters}'");
+                    $escapedFilters = escapeshellarg($filters);
+                    passthru("tail -f {$logPathsStr} | grep -E {$escapedFilters}");
                 } else {
                     passthru("tail -f {$logPathsStr}");
                 }
@@ -68,20 +75,23 @@ class ViewScheduledLogs extends Command
             $this->info("Showing last {$lines} lines of {$logTypeDescription} logs for {$date}{$filterDescription}:");
             $this->line('');
 
+            $escapedLines = escapeshellarg((string) $lines);
             if (count($logPaths) === 1) {
-                $logPath = $logPaths[0];
+                $logPath = escapeshellarg($logPaths[0]);
                 if ($filters) {
-                    passthru("tail -n {$lines} {$logPath} | grep -E '{$filters}'");
+                    $escapedFilters = escapeshellarg($filters);
+                    passthru("tail -n {$escapedLines} {$logPath} | grep -E {$escapedFilters}");
                 } else {
-                    passthru("tail -n {$lines} {$logPath}");
+                    passthru("tail -n {$escapedLines} {$logPath}");
                 }
             } else {
                 // Multiple files - concatenate and sort by timestamp
-                $logPathsStr = implode(' ', $logPaths);
+                $logPathsStr = implode(' ', array_map('escapeshellarg', $logPaths));
                 if ($filters) {
-                    passthru("tail -n {$lines} {$logPathsStr} | sort | grep -E '{$filters}'");
+                    $escapedFilters = escapeshellarg($filters);
+                    passthru("tail -n {$escapedLines} {$logPathsStr} | sort | grep -E {$escapedFilters}");
                 } else {
-                    passthru("tail -n {$lines} {$logPathsStr} | sort");
+                    passthru("tail -n {$escapedLines} {$logPathsStr} | sort");
                 }
             }
         }

app/Console/Kernel.php

@@ -2,26 +2,25 @@
 
 namespace App\Console;
 
-use App\Jobs\CheckAndStartSentinelJob;
+use App\Jobs\ApiTokenExpirationWarningJob;
 use App\Jobs\CheckForUpdatesJob;
 use App\Jobs\CheckHelperImageJob;
+use App\Jobs\CheckTraefikVersionJob;
 use App\Jobs\CleanupInstanceStuffsJob;
+use App\Jobs\CleanupOrphanedPreviewContainersJob;
+use App\Jobs\CleanupStaleMultiplexedConnections;
+use App\Jobs\PullChangelog;
 use App\Jobs\PullTemplatesFromCDN;
 use App\Jobs\RegenerateSslCertJob;
 use App\Jobs\ScheduledJobManager;
-use App\Jobs\ServerResourceManager;
+use App\Jobs\ServerManagerJob;
 use App\Jobs\UpdateCoolifyJob;
 use App\Models\InstanceSettings;
-use App\Models\Server;
-use App\Models\Team;
 use Illuminate\Console\Scheduling\Schedule;
 use Illuminate\Foundation\Console\Kernel as ConsoleKernel;
-use Illuminate\Support\Facades\Log;
 
 class Kernel extends ConsoleKernel
 {
-    private $allServers;
-
     private Schedule $scheduleInstance;
 
     private InstanceSettings $settings;
@@ -33,8 +32,6 @@ class Kernel extends ConsoleKernel
     protected function schedule(Schedule $schedule): void
     {
         $this->scheduleInstance = $schedule;
-        $this->allServers = Server::where('ip', '!=', '1.2.3.4');
-
         $this->settings = instanceSettings();
         $this->updateCheckFrequency = $this->settings->update_check_frequency ?: '0 * * * *';
 
@@ -44,8 +41,13 @@ class Kernel extends ConsoleKernel
             $this->instanceTimezone = config('app.timezone');
         }
 
-        // $this->scheduleInstance->job(new CleanupStaleMultiplexedConnections)->hourly();
-        $this->scheduleInstance->command('cleanup:redis')->weekly();
+        $this->scheduleInstance->call(fn () => app(CleanupStaleMultiplexedConnections::class)->handle())
+            ->name('cleanup:ssh-mux')
+            ->hourly()
+            ->when(fn () => config('constants.ssh.mux_enabled') && ! config('constants.coolify.is_windows_docker_desktop'));
+        $this->scheduleInstance->command('cleanup:redis --clear-locks')->daily();
+        $this->scheduleInstance->command('sanctum:prune-expired --hours=1')->hourly()->onOneServer();
+        $this->scheduleInstance->job(new ApiTokenExpirationWarningJob)->hourly()->onOneServer();
 
         if (isDev()) {
             // Instance Jobs
@@ -54,7 +56,7 @@ class Kernel extends ConsoleKernel
             $this->scheduleInstance->job(new CheckHelperImageJob)->everyTenMinutes()->onOneServer();
 
             // Server Jobs
-            $this->scheduleInstance->job(new ServerResourceManager)->everyMinute()->onOneServer();
+            $this->scheduleInstance->job(new ServerManagerJob)->everyMinute()->onOneServer();
 
             // Scheduled Jobs (Backups & Tasks)
             $this->scheduleInstance->job(new ScheduledJobManager)->everyMinute()->onOneServer();
@@ -67,45 +69,33 @@ class Kernel extends ConsoleKernel
             $this->scheduleInstance->command('cleanup:unreachable-servers')->daily()->onOneServer();
 
             $this->scheduleInstance->job(new PullTemplatesFromCDN)->cron($this->updateCheckFrequency)->timezone($this->instanceTimezone)->onOneServer();
+            $this->scheduleInstance->job(new PullChangelog)->cron($this->updateCheckFrequency)->timezone($this->instanceTimezone)->onOneServer();
 
             $this->scheduleInstance->job(new CleanupInstanceStuffsJob)->everyTwoMinutes()->onOneServer();
             $this->scheduleUpdates();
 
             // Server Jobs
-            $this->scheduleInstance->job(new ServerResourceManager)->everyMinute()->onOneServer();
+            $this->scheduleInstance->job(new ServerManagerJob)->everyMinute()->onOneServer();
 
             $this->pullImages();
 
             // Scheduled Jobs (Backups & Tasks)
             $this->scheduleInstance->job(new ScheduledJobManager)->everyMinute()->onOneServer();
 
-            $this->scheduleInstance->job(new RegenerateSslCertJob)->twiceDaily();
+            $this->scheduleInstance->job(new RegenerateSslCertJob)->twiceDaily()->onOneServer();
+
+            $this->scheduleInstance->job(new CheckTraefikVersionJob)->weekly()->sundays()->at('00:00')->timezone($this->instanceTimezone)->onOneServer();
 
             $this->scheduleInstance->command('cleanup:database --yes')->daily();
             $this->scheduleInstance->command('uploads:clear')->everyTwoMinutes();
+
+            // Cleanup orphaned PR preview containers daily
+            $this->scheduleInstance->job(new CleanupOrphanedPreviewContainersJob)->daily()->onOneServer();
         }
     }
 
     private function pullImages(): void
     {
-        if (isCloud()) {
-            $servers = $this->allServers->whereRelation('team.subscription', 'stripe_invoice_paid', true)->whereRelation('settings', 'is_usable', true)->whereRelation('settings', 'is_reachable', true)->get();
-            $own = Team::find(0)->servers;
-            $servers = $servers->merge($own);
-        } else {
-            $servers = $this->allServers->whereRelation('settings', 'is_usable', true)->whereRelation('settings', 'is_reachable', true)->get();
-        }
-        foreach ($servers as $server) {
-            try {
-                if ($server->isSentinelEnabled()) {
-                    $this->scheduleInstance->job(function () use ($server) {
-                        CheckAndStartSentinelJob::dispatch($server);
-                    })->cron($this->updateCheckFrequency)->timezone($this->instanceTimezone)->onOneServer();
-                }
-            } catch (\Exception $e) {
-                Log::error('Error pulling images: '.$e->getMessage());
-            }
-        }
         $this->scheduleInstance->job(new CheckHelperImageJob)
             ->cron($this->updateCheckFrequency)
             ->timezone($this->instanceTimezone)

app/Data/CoolifyTaskArgs.php

@@ -1,30 +0,0 @@
-<?php
-
-namespace App\Data;
-
-use App\Enums\ProcessStatus;
-use Illuminate\Database\Eloquent\Model;
-use Spatie\LaravelData\Data;
-
-/**
- * The parameters to execute a CoolifyTask, organized in a DTO.
- */
-class CoolifyTaskArgs extends Data
-{
-    public function __construct(
-        public string $server_uuid,
-        public string $command,
-        public string $type,
-        public ?string $type_uuid = null,
-        public ?int $process_id = null,
-        public ?Model $model = null,
-        public ?string $status = null,
-        public bool $ignore_errors = false,
-        public $call_event_on_finish = null,
-        public $call_event_data = null
-    ) {
-        if (is_null($status)) {
-            $this->status = ProcessStatus::QUEUED->value;
-        }
-    }
-}

app/Data/ServerMetadata.php

@@ -10,6 +10,8 @@ class ServerMetadata extends Data
 {
     public function __construct(
         public ?ProxyTypes $type,
-        public ?ProxyStatus $status
+        public ?ProxyStatus $status,
+        public ?string $last_saved_settings = null,
+        public ?string $last_applied_settings = null
     ) {}
 }

app/Enums/BuildPackTypes.php

@@ -8,4 +8,5 @@ enum BuildPackTypes: string
     case STATIC = 'static';
     case DOCKERFILE = 'dockerfile';
     case DOCKERCOMPOSE = 'dockercompose';
+    case RAILPACK = 'railpack';
 }

app/Events/ApplicationConfigurationChanged.php

@@ -0,0 +1,35 @@
+<?php
+
+namespace App\Events;
+
+use Illuminate\Broadcasting\InteractsWithSockets;
+use Illuminate\Broadcasting\PrivateChannel;
+use Illuminate\Contracts\Broadcasting\ShouldBroadcast;
+use Illuminate\Foundation\Events\Dispatchable;
+use Illuminate\Queue\SerializesModels;
+
+class ApplicationConfigurationChanged implements ShouldBroadcast
+{
+    use Dispatchable, InteractsWithSockets, SerializesModels;
+
+    public ?int $teamId = null;
+
+    public function __construct($teamId = null)
+    {
+        if (is_null($teamId) && auth()->check() && auth()->user()->currentTeam()) {
+            $teamId = auth()->user()->currentTeam()->id;
+        }
+        $this->teamId = $teamId;
+    }
+
+    public function broadcastOn(): array
+    {
+        if (is_null($this->teamId)) {
+            return [];
+        }
+
+        return [
+            new PrivateChannel("team.{$this->teamId}"),
+        ];
+    }
+}

app/Events/ProxyStatusChangedUI.php

@@ -14,12 +14,15 @@ class ProxyStatusChangedUI implements ShouldBroadcast
 
     public ?int $teamId = null;
 
-    public function __construct(?int $teamId = null)
+    public ?int $activityId = null;
+
+    public function __construct(?int $teamId = null, ?int $activityId = null)
     {
         if (is_null($teamId) && auth()->check() && auth()->user()->currentTeam()) {
             $teamId = auth()->user()->currentTeam()->id;
         }
         $this->teamId = $teamId;
+        $this->activityId = $activityId;
     }
 
     public function broadcastOn(): array

app/Events/RestoreJobFinished.php

@@ -17,17 +17,23 @@ class RestoreJobFinished
         $tmpPath = data_get($data, 'tmpPath');
         $container = data_get($data, 'container');
         $serverId = data_get($data, 'serverId');
-        if (filled($scriptPath) && filled($tmpPath) && filled($container) && filled($serverId)) {
-            if (str($tmpPath)->startsWith('/tmp/')
-                && str($scriptPath)->startsWith('/tmp/')
-                && ! str($tmpPath)->contains('..')
-                && ! str($scriptPath)->contains('..')
-                && strlen($tmpPath) > 5  // longer than just "/tmp/"
-                && strlen($scriptPath) > 5
-            ) {
-                $commands[] = "docker exec {$container} sh -c 'rm {$scriptPath}'";
-                $commands[] = "docker exec {$container} sh -c 'rm {$tmpPath}'";
-                instant_remote_process($commands, Server::find($serverId), throwError: true);
+
+        if (filled($container) && filled($serverId)) {
+            $commands = [];
+
+            if (isSafeTmpPath($scriptPath)) {
+                $commands[] = 'docker exec '.escapeshellarg($container)." sh -c 'rm ".escapeshellarg($scriptPath)." 2>/dev/null || true'";
+            }
+
+            if (isSafeTmpPath($tmpPath)) {
+                $commands[] = 'docker exec '.escapeshellarg($container)." sh -c 'rm ".escapeshellarg($tmpPath)." 2>/dev/null || true'";
+            }
+
+            if (! empty($commands)) {
+                $server = Server::find($serverId);
+                if ($server) {
+                    instant_remote_process($commands, $server, throwError: false);
+                }
             }
         }
     }

app/Events/S3RestoreJobFinished.php

@@ -0,0 +1,56 @@
+<?php
+
+namespace App\Events;
+
+use App\Models\Server;
+use Illuminate\Broadcasting\InteractsWithSockets;
+use Illuminate\Foundation\Events\Dispatchable;
+use Illuminate\Queue\SerializesModels;
+
+class S3RestoreJobFinished
+{
+    use Dispatchable, InteractsWithSockets, SerializesModels;
+
+    public function __construct($data)
+    {
+        $containerName = data_get($data, 'containerName');
+        $serverTmpPath = data_get($data, 'serverTmpPath');
+        $scriptPath = data_get($data, 'scriptPath');
+        $containerTmpPath = data_get($data, 'containerTmpPath');
+        $container = data_get($data, 'container');
+        $serverId = data_get($data, 'serverId');
+
+        // Most cleanup now happens inline during restore process
+        // This acts as a safety net for edge cases (errors, interruptions)
+        if (filled($serverId)) {
+            $commands = [];
+
+            // Ensure helper container is removed (may already be gone from inline cleanup)
+            if (filled($containerName)) {
+                $commands[] = 'docker rm -f '.escapeshellarg($containerName).' 2>/dev/null || true';
+            }
+
+            // Clean up server temp file if still exists (should already be cleaned)
+            if (isSafeTmpPath($serverTmpPath)) {
+                $commands[] = 'rm -f '.escapeshellarg($serverTmpPath).' 2>/dev/null || true';
+            }
+
+            // Clean up any remaining files in database container (may already be cleaned)
+            if (filled($container)) {
+                if (isSafeTmpPath($containerTmpPath)) {
+                    $commands[] = 'docker exec '.escapeshellarg($container).' rm -f '.escapeshellarg($containerTmpPath).' 2>/dev/null || true';
+                }
+                if (isSafeTmpPath($scriptPath)) {
+                    $commands[] = 'docker exec '.escapeshellarg($container).' rm -f '.escapeshellarg($scriptPath).' 2>/dev/null || true';
+                }
+            }
+
+            if (! empty($commands)) {
+                $server = Server::find($serverId);
+                if ($server) {
+                    instant_remote_process($commands, $server, throwError: false);
+                }
+            }
+        }
+    }
+}

app/Events/SentinelRestarted.php

@@ -0,0 +1,39 @@
+<?php
+
+namespace App\Events;
+
+use App\Models\Server;
+use Illuminate\Broadcasting\InteractsWithSockets;
+use Illuminate\Broadcasting\PrivateChannel;
+use Illuminate\Contracts\Broadcasting\ShouldBroadcast;
+use Illuminate\Foundation\Events\Dispatchable;
+use Illuminate\Queue\SerializesModels;
+
+class SentinelRestarted implements ShouldBroadcast
+{
+    use Dispatchable, InteractsWithSockets, SerializesModels;
+
+    public ?int $teamId = null;
+
+    public ?string $version = null;
+
+    public string $serverUuid;
+
+    public function __construct(Server $server, ?string $version = null)
+    {
+        $this->teamId = $server->team_id;
+        $this->serverUuid = $server->uuid;
+        $this->version = $version;
+    }
+
+    public function broadcastOn(): array
+    {
+        if (is_null($this->teamId)) {
+            return [];
+        }
+
+        return [
+            new PrivateChannel("team.{$this->teamId}"),
+        ];
+    }
+}

app/Events/ServerValidated.php

@@ -0,0 +1,51 @@
+<?php
+
+namespace App\Events;
+
+use Illuminate\Broadcasting\InteractsWithSockets;
+use Illuminate\Broadcasting\PrivateChannel;
+use Illuminate\Contracts\Broadcasting\ShouldBroadcast;
+use Illuminate\Foundation\Events\Dispatchable;
+use Illuminate\Queue\SerializesModels;
+
+class ServerValidated implements ShouldBroadcast
+{
+    use Dispatchable, InteractsWithSockets, SerializesModels;
+
+    public ?int $teamId = null;
+
+    public ?string $serverUuid = null;
+
+    public function __construct(?int $teamId = null, ?string $serverUuid = null)
+    {
+        if (is_null($teamId) && auth()->check() && auth()->user()->currentTeam()) {
+            $teamId = auth()->user()->currentTeam()->id;
+        }
+        $this->teamId = $teamId;
+        $this->serverUuid = $serverUuid;
+    }
+
+    public function broadcastOn(): array
+    {
+        if (is_null($this->teamId)) {
+            return [];
+        }
+
+        return [
+            new PrivateChannel("team.{$this->teamId}"),
+        ];
+    }
+
+    public function broadcastAs(): string
+    {
+        return 'ServerValidated';
+    }
+
+    public function broadcastWith(): array
+    {
+        return [
+            'teamId' => $this->teamId,
+            'serverUuid' => $this->serverUuid,
+        ];
+    }
+}

app/Exceptions/DeploymentException.php

@@ -0,0 +1,32 @@
+<?php
+
+namespace App\Exceptions;
+
+use Exception;
+
+/**
+ * Exception for expected deployment failures caused by user/application errors.
+ * These are not Coolify bugs and should not be logged to laravel.log.
+ * Examples: Nixpacks detection failures, missing Dockerfiles, invalid configs, etc.
+ */
+class DeploymentException extends Exception
+{
+    /**
+     * Create a new deployment exception instance.
+     *
+     * @param  string  $message
+     * @param  int  $code
+     */
+    public function __construct($message = '', $code = 0, ?\Throwable $previous = null)
+    {
+        parent::__construct($message, $code, $previous);
+    }
+
+    /**
+     * Create from another exception, preserving its message and stack trace.
+     */
+    public static function fromException(\Throwable $exception): static
+    {
+        return new static($exception->getMessage(), $exception->getCode(), $exception);
+    }
+}

app/Exceptions/Handler.php

@@ -4,8 +4,10 @@ namespace App\Exceptions;
 
 use App\Models\InstanceSettings;
 use App\Models\User;
+use Illuminate\Auth\Access\AuthorizationException;
 use Illuminate\Auth\AuthenticationException;
 use Illuminate\Foundation\Exceptions\Handler as ExceptionHandler;
+use Psr\Log\LogLevel;
 use RuntimeException;
 use Sentry\Laravel\Integration;
 use Sentry\State\Scope;
@@ -16,7 +18,7 @@ class Handler extends ExceptionHandler
     /**
      * A list of exception types with their corresponding custom log levels.
      *
-     * @var array<class-string<\Throwable>, \Psr\Log\LogLevel::*>
+     * @var array<class-string<Throwable>, LogLevel::*>
      */
     protected $levels = [
         //
@@ -25,10 +27,12 @@ class Handler extends ExceptionHandler
     /**
      * A list of the exception types that are not reported.
      *
-     * @var array<int, class-string<\Throwable>>
+     * @var array<int, class-string<Throwable>>
      */
     protected $dontReport = [
         ProcessException::class,
+        NonReportableException::class,
+        DeploymentException::class,
     ];
 
     /**
@@ -47,12 +51,55 @@ class Handler extends ExceptionHandler
     protected function unauthenticated($request, AuthenticationException $exception)
     {
         if ($request->is('api/*') || $request->expectsJson() || $this->shouldReturnJson($request, $exception)) {
+            if ($request->is('api/*')) {
+                auditLog('api.auth.unauthenticated', [
+                    'reason' => $exception->getMessage(),
+                    'guards' => $exception->guards(),
+                ], 'warning');
+            }
+
             return response()->json(['message' => $exception->getMessage()], 401);
         }
 
         return redirect()->guest($exception->redirectTo($request) ?? route('login'));
     }
 
+    /**
+     * Render an exception into an HTTP response.
+     */
+    public function render($request, Throwable $e)
+    {
+        // Handle authorization exceptions for API routes
+        if ($e instanceof AuthorizationException) {
+            if ($request->is('api/*') || $request->expectsJson()) {
+                if ($request->is('api/*')) {
+                    auditLog('api.auth.policy_denied', [
+                        'reason' => $e->getMessage(),
+                        'route' => $request->route()?->getName() ?? $request->path(),
+                    ], 'warning');
+                }
+
+                // Get the custom message from the policy if available
+                $message = $e->getMessage();
+
+                // Clean up the message for API responses (remove HTML tags if present)
+                $message = strip_tags(str_replace('<br/>', ' ', $message));
+
+                // If no custom message, use a default one
+                if (empty($message) || $message === 'This action is unauthorized.') {
+                    $message = 'You are not authorized to perform this action.';
+                }
+
+                return response()->json([
+                    'message' => $message,
+                    'error' => 'Unauthorized',
+                ], 403);
+            }
+        }
+
+        return parent::render($request, $e);
+    }
+
     /**
      * Register the exception handling callbacks for the application.
      */
@@ -81,9 +128,14 @@ class Handler extends ExceptionHandler
                     );
                 }
             );
+            // Check for errors that should not be reported to Sentry
             if (str($e->getMessage())->contains('No space left on device')) {
+                // Log locally but don't send to Sentry
+                logger()->warning('Disk space error: '.$e->getMessage());
+
                 return;
             }
+
             Integration::captureUnhandledException($e);
         });
     }

app/Exceptions/NonReportableException.php

@@ -0,0 +1,31 @@
+<?php
+
+namespace App\Exceptions;
+
+use Exception;
+
+/**
+ * Exception that should not be reported to Sentry or other error tracking services.
+ * Use this for known, expected errors that don't require external tracking.
+ */
+class NonReportableException extends Exception
+{
+    /**
+     * Create a new non-reportable exception instance.
+     *
+     * @param  string  $message
+     * @param  int  $code
+     */
+    public function __construct($message = '', $code = 0, ?\Throwable $previous = null)
+    {
+        parent::__construct($message, $code, $previous);
+    }
+
+    /**
+     * Create from another exception, preserving its message and stack trace.
+     */
+    public static function fromException(\Throwable $exception): static
+    {
+        return new static($exception->getMessage(), $exception->getCode(), $exception);
+    }
+}

app/Exceptions/RateLimitException.php

@@ -0,0 +1,15 @@
+<?php
+
+namespace App\Exceptions;
+
+use Exception;
+
+class RateLimitException extends Exception
+{
+    public function __construct(
+        string $message = 'Rate limit exceeded.',
+        public readonly ?int $retryAfter = null
+    ) {
+        parent::__construct($message);
+    }
+}