Merge branch 'next' into shadow/fix-docker-time-command

app/Jobs/CheckForUpdatesJob.php

@@ -33,6 +33,9 @@ class CheckForUpdatesJob implements ShouldBeEncrypted, ShouldQueue
                     // New version available
                     $settings->update(['new_version_available' => true]);
                     File::put(base_path('versions.json'), json_encode($versions, JSON_PRETTY_PRINT));
+
+                    // Invalidate cache to ensure fresh data is loaded
+                    invalidate_versions_cache();
                 } else {
                     $settings->update(['new_version_available' => false]);
                 }

app/Jobs/CheckTraefikVersionForServerJob.php

@@ -0,0 +1,173 @@
+<?php
+
+namespace App\Jobs;
+
+use App\Models\Server;
+use App\Notifications\Server\TraefikVersionOutdated;
+use Illuminate\Bus\Queueable;
+use Illuminate\Contracts\Queue\ShouldQueue;
+use Illuminate\Foundation\Bus\Dispatchable;
+use Illuminate\Queue\InteractsWithQueue;
+use Illuminate\Queue\SerializesModels;
+
+class CheckTraefikVersionForServerJob implements ShouldQueue
+{
+    use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
+
+    public $tries = 3;
+
+    public $timeout = 60;
+
+    /**
+     * Create a new job instance.
+     */
+    public function __construct(
+        public Server $server,
+        public array $traefikVersions
+    ) {}
+
+    /**
+     * Execute the job.
+     */
+    public function handle(): void
+    {
+        // Detect current version (makes SSH call)
+        $currentVersion = getTraefikVersionFromDockerCompose($this->server);
+
+        // Update detected version in database
+        $this->server->update(['detected_traefik_version' => $currentVersion]);
+
+        if (! $currentVersion) {
+            return;
+        }
+
+        // Check if image tag is 'latest' by inspecting the image (makes SSH call)
+        $imageTag = instant_remote_process([
+            "docker inspect coolify-proxy --format '{{.Config.Image}}' 2>/dev/null",
+        ], $this->server, false);
+
+        // Handle empty/null response from SSH command
+        if (empty(trim($imageTag))) {
+            return;
+        }
+
+        if (str_contains(strtolower(trim($imageTag)), ':latest')) {
+            return;
+        }
+
+        // Parse current version to extract major.minor.patch
+        $current = ltrim($currentVersion, 'v');
+        if (! preg_match('/^(\d+\.\d+)\.(\d+)$/', $current, $matches)) {
+            return;
+        }
+
+        $currentBranch = $matches[1]; // e.g., "3.6"
+
+        // Find the latest version for this branch
+        $latestForBranch = $this->traefikVersions["v{$currentBranch}"] ?? null;
+
+        if (! $latestForBranch) {
+            // User is on a branch we don't track - check if newer branches exist
+            $newerBranchInfo = $this->getNewerBranchInfo($currentBranch);
+
+            if ($newerBranchInfo) {
+                $this->storeOutdatedInfo($current, $newerBranchInfo['latest'], 'minor_upgrade', $newerBranchInfo['target']);
+            } else {
+                // No newer branch found, clear outdated info
+                $this->server->update(['traefik_outdated_info' => null]);
+            }
+
+            return;
+        }
+
+        // Compare patch version within the same branch
+        $latest = ltrim($latestForBranch, 'v');
+
+        // Always check for newer branches first
+        $newerBranchInfo = $this->getNewerBranchInfo($currentBranch);
+
+        if (version_compare($current, $latest, '<')) {
+            // Patch update available
+            $this->storeOutdatedInfo($current, $latest, 'patch_update', null, $newerBranchInfo);
+        } elseif ($newerBranchInfo) {
+            // Only newer branch available (no patch update)
+            $this->storeOutdatedInfo($current, $newerBranchInfo['latest'], 'minor_upgrade', $newerBranchInfo['target']);
+        } else {
+            // Fully up to date
+            $this->server->update(['traefik_outdated_info' => null]);
+        }
+    }
+
+    /**
+     * Get information about newer branches if available.
+     */
+    private function getNewerBranchInfo(string $currentBranch): ?array
+    {
+        $newestBranch = null;
+        $newestVersion = null;
+
+        foreach ($this->traefikVersions as $branch => $version) {
+            $branchNum = ltrim($branch, 'v');
+            if (version_compare($branchNum, $currentBranch, '>')) {
+                if (! $newestVersion || version_compare($version, $newestVersion, '>')) {
+                    $newestBranch = $branchNum;
+                    $newestVersion = $version;
+                }
+            }
+        }
+
+        if ($newestVersion) {
+            return [
+                'target' => "v{$newestBranch}",
+                'latest' => ltrim($newestVersion, 'v'),
+            ];
+        }
+
+        return null;
+    }
+
+    /**
+     * Store outdated information in database and send immediate notification.
+     */
+    private function storeOutdatedInfo(string $current, string $latest, string $type, ?string $upgradeTarget = null, ?array $newerBranchInfo = null): void
+    {
+        $outdatedInfo = [
+            'current' => $current,
+            'latest' => $latest,
+            'type' => $type,
+            'checked_at' => now()->toIso8601String(),
+        ];
+
+        // For minor upgrades, add the upgrade_target field (e.g., "v3.6")
+        if ($type === 'minor_upgrade' && $upgradeTarget) {
+            $outdatedInfo['upgrade_target'] = $upgradeTarget;
+        }
+
+        // If there's a newer branch available (even for patch updates), include that info
+        if ($newerBranchInfo) {
+            $outdatedInfo['newer_branch_target'] = $newerBranchInfo['target'];
+            $outdatedInfo['newer_branch_latest'] = $newerBranchInfo['latest'];
+        }
+
+        $this->server->update(['traefik_outdated_info' => $outdatedInfo]);
+
+        // Send immediate notification to the team
+        $this->sendNotification($outdatedInfo);
+    }
+
+    /**
+     * Send notification to team about outdated Traefik.
+     */
+    private function sendNotification(array $outdatedInfo): void
+    {
+        // Attach the outdated info as a dynamic property for the notification
+        $this->server->outdatedInfo = $outdatedInfo;
+
+        // Get the team and send notification
+        $team = $this->server->team()->first();
+
+        if ($team) {
+            $team->notify(new TraefikVersionOutdated(collect([$this->server])));
+        }
+    }
+}

app/Jobs/CheckTraefikVersionJob.php

@@ -0,0 +1,45 @@
+<?php
+
+namespace App\Jobs;
+
+use App\Enums\ProxyTypes;
+use App\Models\Server;
+use Illuminate\Bus\Queueable;
+use Illuminate\Contracts\Queue\ShouldQueue;
+use Illuminate\Foundation\Bus\Dispatchable;
+use Illuminate\Queue\InteractsWithQueue;
+use Illuminate\Queue\SerializesModels;
+
+class CheckTraefikVersionJob implements ShouldQueue
+{
+    use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
+
+    public $tries = 3;
+
+    public function handle(): void
+    {
+        // Load versions from cached data
+        $traefikVersions = get_traefik_versions();
+
+        if (empty($traefikVersions)) {
+            return;
+        }
+
+        // Query all servers with Traefik proxy that are reachable
+        $servers = Server::whereNotNull('proxy')
+            ->whereProxyType(ProxyTypes::TRAEFIK->value)
+            ->whereRelation('settings', 'is_reachable', true)
+            ->whereRelation('settings', 'is_usable', true)
+            ->get();
+
+        if ($servers->isEmpty()) {
+            return;
+        }
+
+        // Dispatch individual server check jobs in parallel
+        // Each job will send immediate notifications when outdated Traefik is detected
+        foreach ($servers as $server) {
+            CheckTraefikVersionForServerJob::dispatch($server, $traefikVersions);
+        }
+    }
+}

app/Jobs/CleanupHelperContainersJob.php

@@ -2,6 +2,8 @@
 
 namespace App\Jobs;
 
+use App\Enums\ApplicationDeploymentStatus;
+use App\Models\ApplicationDeploymentQueue;
 use App\Models\Server;
 use Illuminate\Bus\Queueable;
 use Illuminate\Contracts\Queue\ShouldBeEncrypted;
@@ -20,10 +22,51 @@ class CleanupHelperContainersJob implements ShouldBeEncrypted, ShouldBeUnique, S
     public function handle(): void
     {
         try {
+            // Get all active deployments on this server
+            $activeDeployments = ApplicationDeploymentQueue::where('server_id', $this->server->id)
+                ->whereIn('status', [
+                    ApplicationDeploymentStatus::IN_PROGRESS->value,
+                    ApplicationDeploymentStatus::QUEUED->value,
+                ])
+                ->pluck('deployment_uuid')
+                ->toArray();
+
+            \Log::info('CleanupHelperContainersJob - Active deployments', [
+                'server' => $this->server->name,
+                'active_deployment_uuids' => $activeDeployments,
+            ]);
+
             $containers = instant_remote_process_with_timeout(['docker container ps --format \'{{json .}}\' | jq -s \'map(select(.Image | contains("'.config('constants.coolify.registry_url').'/coollabsio/coolify-helper")))\''], $this->server, false);
-            $containerIds = collect(json_decode($containers))->pluck('ID');
-            if ($containerIds->count() > 0) {
-                foreach ($containerIds as $containerId) {
+            $helperContainers = collect(json_decode($containers));
+
+            if ($helperContainers->count() > 0) {
+                foreach ($helperContainers as $container) {
+                    $containerId = data_get($container, 'ID');
+                    $containerName = data_get($container, 'Names');
+
+                    // Check if this container belongs to an active deployment
+                    $isActiveDeployment = false;
+                    foreach ($activeDeployments as $deploymentUuid) {
+                        if (str_contains($containerName, $deploymentUuid)) {
+                            $isActiveDeployment = true;
+                            break;
+                        }
+                    }
+
+                    if ($isActiveDeployment) {
+                        \Log::info('CleanupHelperContainersJob - Skipping active deployment container', [
+                            'container' => $containerName,
+                            'id' => $containerId,
+                        ]);
+
+                        continue;
+                    }
+
+                    \Log::info('CleanupHelperContainersJob - Removing orphaned helper container', [
+                        'container' => $containerName,
+                        'id' => $containerId,
+                    ]);
+
                     instant_remote_process_with_timeout(['docker container rm -f '.$containerId], $this->server, false);
                 }
             }

app/Jobs/CoolifyTask.php

@@ -3,18 +3,35 @@
 namespace App\Jobs;
 
 use App\Actions\CoolifyTask\RunRemoteProcess;
+use App\Enums\ProcessStatus;
 use Illuminate\Bus\Queueable;
 use Illuminate\Contracts\Queue\ShouldBeEncrypted;
 use Illuminate\Contracts\Queue\ShouldQueue;
 use Illuminate\Foundation\Bus\Dispatchable;
 use Illuminate\Queue\InteractsWithQueue;
 use Illuminate\Queue\SerializesModels;
+use Illuminate\Support\Facades\Log;
 use Spatie\Activitylog\Models\Activity;
 
 class CoolifyTask implements ShouldBeEncrypted, ShouldQueue
 {
     use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
 
+    /**
+     * The number of times the job may be attempted.
+     */
+    public $tries = 3;
+
+    /**
+     * The maximum number of unhandled exceptions to allow before failing.
+     */
+    public $maxExceptions = 1;
+
+    /**
+     * The number of seconds the job can run before timing out.
+     */
+    public $timeout = 600;
+
     /**
      * Create a new job instance.
      */
@@ -42,4 +59,53 @@ class CoolifyTask implements ShouldBeEncrypted, ShouldQueue
 
         $remote_process();
     }
+
+    /**
+     * Calculate the number of seconds to wait before retrying the job.
+     */
+    public function backoff(): array
+    {
+        return [30, 90, 180]; // 30s, 90s, 180s between retries
+    }
+
+    /**
+     * Handle a job failure.
+     */
+    public function failed(?\Throwable $exception): void
+    {
+        Log::channel('scheduled-errors')->error('CoolifyTask permanently failed', [
+            'job' => 'CoolifyTask',
+            'activity_id' => $this->activity->id,
+            'server_uuid' => $this->activity->getExtraProperty('server_uuid'),
+            'command_preview' => substr($this->activity->getExtraProperty('command') ?? '', 0, 200),
+            'error' => $exception?->getMessage(),
+            'total_attempts' => $this->attempts(),
+            'trace' => $exception?->getTraceAsString(),
+        ]);
+
+        // Update activity status to reflect permanent failure
+        $this->activity->properties = $this->activity->properties->merge([
+            'status' => ProcessStatus::ERROR->value,
+            'error' => $exception?->getMessage() ?? 'Job permanently failed',
+            'failed_at' => now()->toIso8601String(),
+        ]);
+        $this->activity->save();
+
+        // Dispatch cleanup event on failure (same as on success)
+        if ($this->call_event_on_finish) {
+            try {
+                $eventClass = "App\\Events\\$this->call_event_on_finish";
+                if (! is_null($this->call_event_data)) {
+                    event(new $eventClass($this->call_event_data));
+                } else {
+                    event(new $eventClass($this->activity->causer_id));
+                }
+                Log::info('Cleanup event dispatched after job failure', [
+                    'event' => $this->call_event_on_finish,
+                ]);
+            } catch (\Throwable $e) {
+                Log::error('Error dispatching cleanup event on failure: '.$e->getMessage());
+            }
+        }
+    }
 }

app/Jobs/DatabaseBackupJob.php

@@ -23,6 +23,7 @@ use Illuminate\Contracts\Queue\ShouldQueue;
 use Illuminate\Foundation\Bus\Dispatchable;
 use Illuminate\Queue\InteractsWithQueue;
 use Illuminate\Queue\SerializesModels;
+use Illuminate\Support\Facades\Log;
 use Illuminate\Support\Str;
 use Throwable;
 use Visus\Cuid2\Cuid2;
@@ -31,6 +32,8 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue
 {
     use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
 
+    public $maxExceptions = 1;
+
     public ?Team $team = null;
 
     public Server $server;
@@ -74,7 +77,7 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue
     public function __construct(public ScheduledDatabaseBackup $backup)
     {
         $this->onQueue('high');
-        $this->timeout = $backup->timeout;
+        $this->timeout = $backup->timeout ?? 3600;
     }
 
     public function handle(): void
@@ -486,17 +489,22 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue
                     $collectionsToExclude = collect();
                 }
                 $commands[] = 'mkdir -p '.$this->backup_dir;
+
+                // Validate and escape database name to prevent command injection
+                validateShellSafePath($databaseName, 'database name');
+                $escapedDatabaseName = escapeshellarg($databaseName);
+
                 if ($collectionsToExclude->count() === 0) {
                     if (str($this->database->image)->startsWith('mongo:4')) {
                         $commands[] = "docker exec $this->container_name mongodump --uri=\"$url\" --gzip --archive > $this->backup_location";
                     } else {
-                        $commands[] = "docker exec $this->container_name mongodump --authenticationDatabase=admin --uri=\"$url\" --db $databaseName --gzip --archive > $this->backup_location";
+                        $commands[] = "docker exec $this->container_name mongodump --authenticationDatabase=admin --uri=\"$url\" --db $escapedDatabaseName --gzip --archive > $this->backup_location";
                     }
                 } else {
                     if (str($this->database->image)->startsWith('mongo:4')) {
                         $commands[] = "docker exec $this->container_name mongodump --uri=$url --gzip --excludeCollection ".$collectionsToExclude->implode(' --excludeCollection ')." --archive > $this->backup_location";
                     } else {
-                        $commands[] = "docker exec $this->container_name mongodump --authenticationDatabase=admin --uri=\"$url\" --db $databaseName --gzip --excludeCollection ".$collectionsToExclude->implode(' --excludeCollection ')." --archive > $this->backup_location";
+                        $commands[] = "docker exec $this->container_name mongodump --authenticationDatabase=admin --uri=\"$url\" --db $escapedDatabaseName --gzip --excludeCollection ".$collectionsToExclude->implode(' --excludeCollection ')." --archive > $this->backup_location";
                     }
                 }
             }
@@ -522,7 +530,10 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue
             if ($this->backup->dump_all) {
                 $backupCommand .= " $this->container_name pg_dumpall --username {$this->database->postgres_user} | gzip > $this->backup_location";
             } else {
-                $backupCommand .= " $this->container_name pg_dump --format=custom --no-acl --no-owner --username {$this->database->postgres_user} $database > $this->backup_location";
+                // Validate and escape database name to prevent command injection
+                validateShellSafePath($database, 'database name');
+                $escapedDatabase = escapeshellarg($database);
+                $backupCommand .= " $this->container_name pg_dump --format=custom --no-acl --no-owner --username {$this->database->postgres_user} $escapedDatabase > $this->backup_location";
             }
 
             $commands[] = $backupCommand;
@@ -544,7 +555,10 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue
             if ($this->backup->dump_all) {
                 $commands[] = "docker exec $this->container_name mysqldump -u root -p\"{$this->database->mysql_root_password}\" --all-databases --single-transaction --quick --lock-tables=false --compress | gzip > $this->backup_location";
             } else {
-                $commands[] = "docker exec $this->container_name mysqldump -u root -p\"{$this->database->mysql_root_password}\" $database > $this->backup_location";
+                // Validate and escape database name to prevent command injection
+                validateShellSafePath($database, 'database name');
+                $escapedDatabase = escapeshellarg($database);
+                $commands[] = "docker exec $this->container_name mysqldump -u root -p\"{$this->database->mysql_root_password}\" $escapedDatabase > $this->backup_location";
             }
             $this->backup_output = instant_remote_process($commands, $this->server);
             $this->backup_output = trim($this->backup_output);
@@ -564,7 +578,10 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue
             if ($this->backup->dump_all) {
                 $commands[] = "docker exec $this->container_name mariadb-dump -u root -p\"{$this->database->mariadb_root_password}\" --all-databases --single-transaction --quick --lock-tables=false --compress > $this->backup_location";
             } else {
-                $commands[] = "docker exec $this->container_name mariadb-dump -u root -p\"{$this->database->mariadb_root_password}\" $database > $this->backup_location";
+                // Validate and escape database name to prevent command injection
+                validateShellSafePath($database, 'database name');
+                $escapedDatabase = escapeshellarg($database);
+                $commands[] = "docker exec $this->container_name mariadb-dump -u root -p\"{$this->database->mariadb_root_password}\" $escapedDatabase > $this->backup_location";
             }
             $this->backup_output = instant_remote_process($commands, $this->server);
             $this->backup_output = trim($this->backup_output);
@@ -636,7 +653,13 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue
             } else {
                 $commands[] = "docker run -d --network {$network} --name backup-of-{$this->backup_log_uuid} --rm -v $this->backup_location:$this->backup_location:ro {$fullImageName}";
             }
-            $commands[] = "docker exec backup-of-{$this->backup_log_uuid} mc alias set temporary {$endpoint} {$key} \"{$secret}\"";
+
+            // Escape S3 credentials to prevent command injection
+            $escapedEndpoint = escapeshellarg($endpoint);
+            $escapedKey = escapeshellarg($key);
+            $escapedSecret = escapeshellarg($secret);
+
+            $commands[] = "docker exec backup-of-{$this->backup_log_uuid} mc alias set temporary {$escapedEndpoint} {$escapedKey} {$escapedSecret}";
             $commands[] = "docker exec backup-of-{$this->backup_log_uuid} mc cp $this->backup_location temporary/$bucket{$this->backup_dir}/";
             instant_remote_process($commands, $this->server);
 
@@ -661,15 +684,34 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue
 
     public function failed(?Throwable $exception): void
     {
+        Log::channel('scheduled-errors')->error('DatabaseBackup permanently failed', [
+            'job' => 'DatabaseBackupJob',
+            'backup_id' => $this->backup->uuid,
+            'database' => $this->database?->name ?? 'unknown',
+            'database_type' => get_class($this->database ?? new \stdClass),
+            'server' => $this->server?->name ?? 'unknown',
+            'total_attempts' => $this->attempts(),
+            'error' => $exception?->getMessage(),
+            'trace' => $exception?->getTraceAsString(),
+        ]);
+
         $log = ScheduledDatabaseBackupExecution::where('uuid', $this->backup_log_uuid)->first();
 
         if ($log) {
             $log->update([
                 'status' => 'failed',
-                'message' => 'Job failed: '.($exception?->getMessage() ?? 'Unknown error'),
+                'message' => 'Job permanently failed after '.$this->attempts().' attempts: '.($exception?->getMessage() ?? 'Unknown error'),
                 'size' => 0,
                 'filename' => null,
+                'finished_at' => Carbon::now(),
             ]);
         }
+
+        // Notify team about permanent failure
+        if ($this->team) {
+            $databaseName = $log?->database_name ?? 'unknown';
+            $output = $this->backup_output ?? $exception?->getMessage() ?? 'Unknown error';
+            $this->team->notify(new BackupFailed($this->backup, $this->database, $output, $databaseName));
+        }
     }
 }

app/Jobs/DeleteResourceJob.php

@@ -124,16 +124,54 @@ class DeleteResourceJob implements ShouldBeEncrypted, ShouldQueue
             $this->resource->delete();
         }
 
+        // Cancel any active deployments for this PR (same logic as API cancel_deployment)
+        $activeDeployments = \App\Models\ApplicationDeploymentQueue::where('application_id', $application->id)
+            ->where('pull_request_id', $pull_request_id)
+            ->whereIn('status', [
+                \App\Enums\ApplicationDeploymentStatus::QUEUED->value,
+                \App\Enums\ApplicationDeploymentStatus::IN_PROGRESS->value,
+            ])
+            ->get();
+
+        foreach ($activeDeployments as $activeDeployment) {
+            try {
+                // Mark deployment as cancelled
+                $activeDeployment->update([
+                    'status' => \App\Enums\ApplicationDeploymentStatus::CANCELLED_BY_USER->value,
+                ]);
+
+                // Add cancellation log entry
+                $activeDeployment->addLogEntry('Deployment cancelled: Pull request closed.', 'stderr');
+
+                // Check if helper container exists and kill it
+                $deployment_uuid = $activeDeployment->deployment_uuid;
+                $escapedDeploymentUuid = escapeshellarg($deployment_uuid);
+                $checkCommand = "docker ps -a --filter name={$escapedDeploymentUuid} --format '{{.Names}}'";
+                $containerExists = instant_remote_process([$checkCommand], $server);
+
+                if ($containerExists && str($containerExists)->trim()->isNotEmpty()) {
+                    instant_remote_process(["docker rm -f {$escapedDeploymentUuid}"], $server);
+                    $activeDeployment->addLogEntry('Deployment container stopped.');
+                } else {
+                    $activeDeployment->addLogEntry('Helper container not yet started. Deployment will be cancelled when job checks status.');
+                }
+
+            } catch (\Throwable $e) {
+                // Silently handle errors during deployment cancellation
+            }
+        }
+
         try {
             if ($server->isSwarm()) {
-                instant_remote_process(["docker stack rm {$application->uuid}-{$pull_request_id}"], $server);
+                $escapedStackName = escapeshellarg("{$application->uuid}-{$pull_request_id}");
+                instant_remote_process(["docker stack rm {$escapedStackName}"], $server);
             } else {
                 $containers = getCurrentApplicationContainerStatus($server, $application->id, $pull_request_id)->toArray();
                 $this->stopPreviewContainers($containers, $server);
             }
         } catch (\Throwable $e) {
             // Log the error but don't fail the job
-            ray('Error stopping preview containers: '.$e->getMessage());
+            \Log::warning('Error stopping preview containers for application '.$application->uuid.', PR #'.$pull_request_id.': '.$e->getMessage());
         }
 
         // Finally, force delete to trigger resource cleanup
@@ -156,7 +194,6 @@ class DeleteResourceJob implements ShouldBeEncrypted, ShouldQueue
             "docker stop -t $timeout $containerList",
             "docker rm -f $containerList",
         ];
-
         instant_remote_process(
             command: $commands,
             server: $server,

app/Jobs/PullHelperImageJob.php

@@ -1,30 +0,0 @@
-<?php
-
-namespace App\Jobs;
-
-use App\Models\Server;
-use Illuminate\Bus\Queueable;
-use Illuminate\Contracts\Queue\ShouldBeEncrypted;
-use Illuminate\Contracts\Queue\ShouldQueue;
-use Illuminate\Foundation\Bus\Dispatchable;
-use Illuminate\Queue\InteractsWithQueue;
-use Illuminate\Queue\SerializesModels;
-
-class PullHelperImageJob implements ShouldBeEncrypted, ShouldQueue
-{
-    use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
-
-    public $timeout = 1000;
-
-    public function __construct(public Server $server)
-    {
-        $this->onQueue('high');
-    }
-
-    public function handle(): void
-    {
-        $helperImage = config('constants.coolify.helper_image');
-        $latest_version = getHelperVersion();
-        instant_remote_process(["docker pull -q {$helperImage}:{$latest_version}"], $this->server, false);
-    }
-}

app/Jobs/PushServerUpdateJob.php

@@ -13,6 +13,8 @@ use App\Models\Server;
 use App\Models\ServiceApplication;
 use App\Models\ServiceDatabase;
 use App\Notifications\Container\ContainerRestarted;
+use App\Services\ContainerStatusAggregator;
+use App\Traits\CalculatesExcludedStatus;
 use Illuminate\Bus\Queueable;
 use Illuminate\Contracts\Queue\ShouldBeEncrypted;
 use Illuminate\Contracts\Queue\ShouldQueue;
@@ -25,6 +27,7 @@ use Laravel\Horizon\Contracts\Silenced;
 
 class PushServerUpdateJob implements ShouldBeEncrypted, ShouldQueue, Silenced
 {
+    use CalculatesExcludedStatus;
     use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
 
     public $tries = 1;
@@ -67,6 +70,8 @@ class PushServerUpdateJob implements ShouldBeEncrypted, ShouldQueue, Silenced
 
     public Collection $applicationContainerStatuses;
 
+    public Collection $serviceContainerStatuses;
+
     public bool $foundProxy = false;
 
     public bool $foundLogDrainContainer = false;
@@ -90,6 +95,7 @@ class PushServerUpdateJob implements ShouldBeEncrypted, ShouldQueue, Silenced
         $this->foundApplicationPreviewsIds = collect();
         $this->foundServiceDatabaseIds = collect();
         $this->applicationContainerStatuses = collect();
+        $this->serviceContainerStatuses = collect();
         $this->allApplicationIds = collect();
         $this->allDatabaseUuids = collect();
         $this->allTcpProxyUuids = collect();
@@ -99,6 +105,20 @@ class PushServerUpdateJob implements ShouldBeEncrypted, ShouldQueue, Silenced
 
     public function handle()
     {
+        // Defensive initialization for Collection properties to handle queue deserialization edge cases
+        $this->serviceContainerStatuses ??= collect();
+        $this->applicationContainerStatuses ??= collect();
+        $this->foundApplicationIds ??= collect();
+        $this->foundDatabaseUuids ??= collect();
+        $this->foundServiceApplicationIds ??= collect();
+        $this->foundApplicationPreviewsIds ??= collect();
+        $this->foundServiceDatabaseIds ??= collect();
+        $this->allApplicationIds ??= collect();
+        $this->allDatabaseUuids ??= collect();
+        $this->allTcpProxyUuids ??= collect();
+        $this->allServiceApplicationIds ??= collect();
+        $this->allServiceDatabaseIds ??= collect();
+
         // TODO: Swarm is not supported yet
         if (! $this->data) {
             throw new \Exception('No data provided');
@@ -108,7 +128,6 @@ class PushServerUpdateJob implements ShouldBeEncrypted, ShouldQueue, Silenced
         $this->server->sentinelHeartbeat();
 
         $this->containers = collect(data_get($data, 'containers'));
-
         $filesystemUsageRoot = data_get($data, 'filesystem_usage_root.used_percentage');
         ServerStorageCheckJob::dispatch($this->server, $filesystemUsageRoot);
 
@@ -141,65 +160,88 @@ class PushServerUpdateJob implements ShouldBeEncrypted, ShouldQueue, Silenced
 
         foreach ($this->containers as $container) {
             $containerStatus = data_get($container, 'state', 'exited');
-            $containerHealth = data_get($container, 'health_status', 'unhealthy');
-            $containerStatus = "$containerStatus ($containerHealth)";
+            $rawHealthStatus = data_get($container, 'health_status');
+            $containerHealth = $rawHealthStatus ?? 'unknown';
+            // Only append health status if container is not exited
+            if ($containerStatus !== 'exited') {
+                $containerStatus = "$containerStatus:$containerHealth";
+            }
             $labels = collect(data_get($container, 'labels'));
             $coolify_managed = $labels->has('coolify.managed');
-            if ($coolify_managed) {
-                $name = data_get($container, 'name');
-                if ($name === 'coolify-log-drain' && $this->isRunning($containerStatus)) {
-                    $this->foundLogDrainContainer = true;
-                }
-                if ($labels->has('coolify.applicationId')) {
-                    $applicationId = $labels->get('coolify.applicationId');
-                    $pullRequestId = $labels->get('coolify.pullRequestId', '0');
-                    try {
-                        if ($pullRequestId === '0') {
-                            if ($this->allApplicationIds->contains($applicationId) && $this->isRunning($containerStatus)) {
-                                $this->foundApplicationIds->push($applicationId);
-                            }
-                            // Store container status for aggregation
-                            if (! $this->applicationContainerStatuses->has($applicationId)) {
-                                $this->applicationContainerStatuses->put($applicationId, collect());
-                            }
-                            $containerName = $labels->get('com.docker.compose.service');
-                            if ($containerName) {
-                                $this->applicationContainerStatuses->get($applicationId)->put($containerName, $containerStatus);
-                            }
-                        } else {
-                            $previewKey = $applicationId.':'.$pullRequestId;
-                            if ($this->allApplicationPreviewsIds->contains($previewKey) && $this->isRunning($containerStatus)) {
-                                $this->foundApplicationPreviewsIds->push($previewKey);
-                            }
-                            $this->updateApplicationPreviewStatus($applicationId, $pullRequestId, $containerStatus);
+
+            if (! $coolify_managed) {
+                continue;
+            }
+
+            $name = data_get($container, 'name');
+            if ($name === 'coolify-log-drain' && $this->isRunning($containerStatus)) {
+                $this->foundLogDrainContainer = true;
+            }
+            if ($labels->has('coolify.applicationId')) {
+                $applicationId = $labels->get('coolify.applicationId');
+                $pullRequestId = $labels->get('coolify.pullRequestId', '0');
+                try {
+                    if ($pullRequestId === '0') {
+                        if ($this->allApplicationIds->contains($applicationId)) {
+                            $this->foundApplicationIds->push($applicationId);
+                        }
+                        // Store container status for aggregation
+                        if (! $this->applicationContainerStatuses->has($applicationId)) {
+                            $this->applicationContainerStatuses->put($applicationId, collect());
+                        }
+                        $containerName = $labels->get('com.docker.compose.service');
+                        if ($containerName) {
+                            $this->applicationContainerStatuses->get($applicationId)->put($containerName, $containerStatus);
                         }
-                    } catch (\Exception $e) {
-                    }
-                } elseif ($labels->has('coolify.serviceId')) {
-                    $serviceId = $labels->get('coolify.serviceId');
-                    $subType = $labels->get('coolify.service.subType');
-                    $subId = $labels->get('coolify.service.subId');
-                    if ($subType === 'application' && $this->isRunning($containerStatus)) {
-                        $this->foundServiceApplicationIds->push($subId);
-                        $this->updateServiceSubStatus($serviceId, $subType, $subId, $containerStatus);
-                    } elseif ($subType === 'database' && $this->isRunning($containerStatus)) {
-                        $this->foundServiceDatabaseIds->push($subId);
-                        $this->updateServiceSubStatus($serviceId, $subType, $subId, $containerStatus);
-                    }
-                } else {
-                    $uuid = $labels->get('com.docker.compose.service');
-                    $type = $labels->get('coolify.type');
-                    if ($name === 'coolify-proxy' && $this->isRunning($containerStatus)) {
-                        $this->foundProxy = true;
-                    } elseif ($type === 'service' && $this->isRunning($containerStatus)) {
                     } else {
-                        if ($this->allDatabaseUuids->contains($uuid) && $this->isRunning($containerStatus)) {
-                            $this->foundDatabaseUuids->push($uuid);
-                            if ($this->allTcpProxyUuids->contains($uuid) && $this->isRunning($containerStatus)) {
-                                $this->updateDatabaseStatus($uuid, $containerStatus, tcpProxy: true);
-                            } else {
-                                $this->updateDatabaseStatus($uuid, $containerStatus, tcpProxy: false);
-                            }
+                        $previewKey = $applicationId.':'.$pullRequestId;
+                        if ($this->allApplicationPreviewsIds->contains($previewKey)) {
+                            $this->foundApplicationPreviewsIds->push($previewKey);
+                        }
+                        $this->updateApplicationPreviewStatus($applicationId, $pullRequestId, $containerStatus);
+                    }
+                } catch (\Exception $e) {
+                }
+            } elseif ($labels->has('coolify.serviceId')) {
+                $serviceId = $labels->get('coolify.serviceId');
+                $subType = $labels->get('coolify.service.subType');
+                $subId = $labels->get('coolify.service.subId');
+                if ($subType === 'application') {
+                    $this->foundServiceApplicationIds->push($subId);
+                    // Store container status for aggregation
+                    $key = $serviceId.':'.$subType.':'.$subId;
+                    if (! $this->serviceContainerStatuses->has($key)) {
+                        $this->serviceContainerStatuses->put($key, collect());
+                    }
+                    $containerName = $labels->get('com.docker.compose.service');
+                    if ($containerName) {
+                        $this->serviceContainerStatuses->get($key)->put($containerName, $containerStatus);
+                    }
+                } elseif ($subType === 'database') {
+                    $this->foundServiceDatabaseIds->push($subId);
+                    // Store container status for aggregation
+                    $key = $serviceId.':'.$subType.':'.$subId;
+                    if (! $this->serviceContainerStatuses->has($key)) {
+                        $this->serviceContainerStatuses->put($key, collect());
+                    }
+                    $containerName = $labels->get('com.docker.compose.service');
+                    if ($containerName) {
+                        $this->serviceContainerStatuses->get($key)->put($containerName, $containerStatus);
+                    }
+                }
+            } else {
+                $uuid = $labels->get('com.docker.compose.service');
+                $type = $labels->get('coolify.type');
+                if ($name === 'coolify-proxy' && $this->isRunning($containerStatus)) {
+                    $this->foundProxy = true;
+                } elseif ($type === 'service' && $this->isRunning($containerStatus)) {
+                } else {
+                    if ($this->allDatabaseUuids->contains($uuid) && $this->isRunning($containerStatus)) {
+                        $this->foundDatabaseUuids->push($uuid);
+                        if ($this->allTcpProxyUuids->contains($uuid) && $this->isRunning($containerStatus)) {
+                            $this->updateDatabaseStatus($uuid, $containerStatus, tcpProxy: true);
+                        } else {
+                            $this->updateDatabaseStatus($uuid, $containerStatus, tcpProxy: false);
                         }
                     }
                 }
@@ -218,6 +260,9 @@ class PushServerUpdateJob implements ShouldBeEncrypted, ShouldQueue, Silenced
         // Aggregate multi-container application statuses
         $this->aggregateMultiContainerStatuses();
 
+        // Aggregate multi-container service statuses
+        $this->aggregateServiceContainerStatuses();
+
         $this->checkLogDrainContainer();
     }
 
@@ -235,57 +280,28 @@ class PushServerUpdateJob implements ShouldBeEncrypted, ShouldQueue, Silenced
 
             // Parse docker compose to check for excluded containers
             $dockerComposeRaw = data_get($application, 'docker_compose_raw');
-            $excludedContainers = collect();
-
-            if ($dockerComposeRaw) {
-                try {
-                    $dockerCompose = \Symfony\Component\Yaml\Yaml::parse($dockerComposeRaw);
-                    $services = data_get($dockerCompose, 'services', []);
-
-                    foreach ($services as $serviceName => $serviceConfig) {
-                        // Check if container should be excluded
-                        $excludeFromHc = data_get($serviceConfig, 'exclude_from_hc', false);
-                        $restartPolicy = data_get($serviceConfig, 'restart', 'always');
-
-                        if ($excludeFromHc || $restartPolicy === 'no') {
-                            $excludedContainers->push($serviceName);
-                        }
-                    }
-                } catch (\Exception $e) {
-                    // If we can't parse, treat all containers as included
-                }
-            }
+            $excludedContainers = $this->getExcludedContainersFromDockerCompose($dockerComposeRaw);
 
             // Filter out excluded containers
             $relevantStatuses = $containerStatuses->filter(function ($status, $containerName) use ($excludedContainers) {
                 return ! $excludedContainers->contains($containerName);
             });
 
-            // If all containers are excluded, don't update status
+            // If all containers are excluded, calculate status from excluded containers
             if ($relevantStatuses->isEmpty()) {
+                $aggregatedStatus = $this->calculateExcludedStatusFromStrings($containerStatuses);
+
+                if ($aggregatedStatus && $application->status !== $aggregatedStatus) {
+                    $application->status = $aggregatedStatus;
+                    $application->save();
+                }
+
                 continue;
             }
 
-            // Aggregate status: if any container is running, app is running
-            $hasRunning = false;
-            $hasUnhealthy = false;
-
-            foreach ($relevantStatuses as $status) {
-                if (str($status)->contains('running')) {
-                    $hasRunning = true;
-                    if (str($status)->contains('unhealthy')) {
-                        $hasUnhealthy = true;
-                    }
-                }
-            }
-
-            $aggregatedStatus = null;
-            if ($hasRunning) {
-                $aggregatedStatus = $hasUnhealthy ? 'running (unhealthy)' : 'running (healthy)';
-            } else {
-                // All containers are exited
-                $aggregatedStatus = 'exited (unhealthy)';
-            }
+            // Use ContainerStatusAggregator service for state machine logic
+            $aggregator = new ContainerStatusAggregator;
+            $aggregatedStatus = $aggregator->aggregateFromStrings($relevantStatuses, 0);
 
             // Update application status with aggregated result
             if ($aggregatedStatus && $application->status !== $aggregatedStatus) {
@@ -295,6 +311,66 @@ class PushServerUpdateJob implements ShouldBeEncrypted, ShouldQueue, Silenced
         }
     }
 
+    private function aggregateServiceContainerStatuses()
+    {
+        if ($this->serviceContainerStatuses->isEmpty()) {
+            return;
+        }
+
+        foreach ($this->serviceContainerStatuses as $key => $containerStatuses) {
+            // Parse key: serviceId:subType:subId
+            [$serviceId, $subType, $subId] = explode(':', $key);
+
+            $service = $this->services->where('id', $serviceId)->first();
+            if (! $service) {
+                continue;
+            }
+
+            // Get the service sub-resource (ServiceApplication or ServiceDatabase)
+            $subResource = null;
+            if ($subType === 'application') {
+                $subResource = $service->applications()->where('id', $subId)->first();
+            } elseif ($subType === 'database') {
+                $subResource = $service->databases()->where('id', $subId)->first();
+            }
+
+            if (! $subResource) {
+                continue;
+            }
+
+            // Parse docker compose from service to check for excluded containers
+            $dockerComposeRaw = data_get($service, 'docker_compose_raw');
+            $excludedContainers = $this->getExcludedContainersFromDockerCompose($dockerComposeRaw);
+
+            // Filter out excluded containers
+            $relevantStatuses = $containerStatuses->filter(function ($status, $containerName) use ($excludedContainers) {
+                return ! $excludedContainers->contains($containerName);
+            });
+
+            // If all containers are excluded, calculate status from excluded containers
+            if ($relevantStatuses->isEmpty()) {
+                $aggregatedStatus = $this->calculateExcludedStatusFromStrings($containerStatuses);
+                if ($aggregatedStatus && $subResource->status !== $aggregatedStatus) {
+                    $subResource->status = $aggregatedStatus;
+                    $subResource->save();
+                }
+
+                continue;
+            }
+
+            // Use ContainerStatusAggregator service for state machine logic
+            // NOTE: Sentinel does NOT provide restart count data, so maxRestartCount is always 0
+            $aggregator = new ContainerStatusAggregator;
+            $aggregatedStatus = $aggregator->aggregateFromStrings($relevantStatuses, 0);
+
+            // Update service sub-resource status with aggregated result
+            if ($aggregatedStatus && $subResource->status !== $aggregatedStatus) {
+                $subResource->status = $aggregatedStatus;
+                $subResource->save();
+            }
+        }
+    }
+
     private function updateApplicationStatus(string $applicationId, string $containerStatus)
     {
         $application = $this->applications->where('id', $applicationId)->first();

app/Jobs/RestartProxyJob.php

@@ -31,12 +31,12 @@ class RestartProxyJob implements ShouldBeEncrypted, ShouldQueue
     public function handle()
     {
         try {
-            StopProxy::run($this->server);
+            StopProxy::run($this->server, restarting: true);
 
             $this->server->proxy->force_stop = false;
             $this->server->save();
 
-            StartProxy::run($this->server, force: true);
+            StartProxy::run($this->server, force: true, restarting: true);
 
         } catch (\Throwable $e) {
             return handleError($e);

app/Jobs/ScheduledJobManager.php

@@ -52,7 +52,7 @@ class ScheduledJobManager implements ShouldQueue
     {
         return [
             (new WithoutOverlapping('scheduled-job-manager'))
-                ->expireAfter(60)   // Lock expires after 1 minute to prevent stale locks
+                ->expireAfter(90)   // Lock expires after 90s to handle high-load environments with many tasks
                 ->dontRelease(),    // Don't re-queue on lock conflict
         ];
     }

app/Jobs/ScheduledTaskJob.php

@@ -18,14 +18,30 @@ use Illuminate\Contracts\Queue\ShouldQueue;
 use Illuminate\Foundation\Bus\Dispatchable;
 use Illuminate\Queue\InteractsWithQueue;
 use Illuminate\Queue\SerializesModels;
+use Illuminate\Support\Facades\Log;
 
 class ScheduledTaskJob implements ShouldQueue
 {
     use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
 
+    /**
+     * The number of times the job may be attempted.
+     */
+    public $tries = 3;
+
+    /**
+     * The maximum number of unhandled exceptions to allow before failing.
+     */
+    public $maxExceptions = 1;
+
+    /**
+     * The number of seconds the job can run before timing out.
+     */
+    public $timeout = 300;
+
     public Team $team;
 
-    public Server $server;
+    public ?Server $server = null;
 
     public ScheduledTask $task;
 
@@ -33,6 +49,11 @@ class ScheduledTaskJob implements ShouldQueue
 
     public ?ScheduledTaskExecution $task_log = null;
 
+    /**
+     * Store execution ID to survive job serialization for timeout handling.
+     */
+    protected ?int $executionId = null;
+
     public string $task_status = 'failed';
 
     public ?string $task_output = null;
@@ -55,6 +76,9 @@ class ScheduledTaskJob implements ShouldQueue
         }
         $this->team = Team::findOrFail($task->team_id);
         $this->server_timezone = $this->getServerTimezone();
+
+        // Set timeout from task configuration
+        $this->timeout = $this->task->timeout ?? 300;
     }
 
     private function getServerTimezone(): string
@@ -70,11 +94,18 @@ class ScheduledTaskJob implements ShouldQueue
 
     public function handle(): void
     {
+        $startTime = Carbon::now();
+
         try {
             $this->task_log = ScheduledTaskExecution::create([
                 'scheduled_task_id' => $this->task->id,
+                'started_at' => $startTime,
+                'retry_count' => $this->attempts() - 1,
             ]);
 
+            // Store execution ID for timeout handling
+            $this->executionId = $this->task_log->id;
+
             $this->server = $this->resource->destination->server;
 
             if ($this->resource->type() === 'application') {
@@ -129,15 +160,101 @@ class ScheduledTaskJob implements ShouldQueue
                     'message' => $this->task_output ?? $e->getMessage(),
                 ]);
             }
-            $this->team?->notify(new TaskFailed($this->task, $e->getMessage()));
+
+            // Log the error to the scheduled-errors channel
+            Log::channel('scheduled-errors')->error('ScheduledTask execution failed', [
+                'job' => 'ScheduledTaskJob',
+                'task_id' => $this->task->uuid,
+                'task_name' => $this->task->name,
+                'server' => $this->server?->name ?? 'unknown',
+                'attempt' => $this->attempts(),
+                'error' => $e->getMessage(),
+            ]);
+
+            // Only notify and throw on final failure
+
+            // Re-throw to trigger Laravel's retry mechanism with backoff
             throw $e;
         } finally {
             ScheduledTaskDone::dispatch($this->team->id);
             if ($this->task_log) {
+                $finishedAt = Carbon::now();
+                $duration = round($startTime->floatDiffInSeconds($finishedAt), 2);
+
                 $this->task_log->update([
-                    'finished_at' => Carbon::now()->toImmutable(),
+                    'finished_at' => $finishedAt->toImmutable(),
+                    'duration' => $duration,
                 ]);
             }
         }
     }
+
+    /**
+     * Calculate the number of seconds to wait before retrying the job.
+     */
+    public function backoff(): array
+    {
+        return [30, 60, 120]; // 30s, 60s, 120s between retries
+    }
+
+    /**
+     * Handle a job failure.
+     */
+    public function failed(?\Throwable $exception): void
+    {
+        Log::channel('scheduled-errors')->error('ScheduledTask permanently failed', [
+            'job' => 'ScheduledTaskJob',
+            'task_id' => $this->task->uuid,
+            'task_name' => $this->task->name,
+            'server' => $this->server?->name ?? 'unknown',
+            'total_attempts' => $this->attempts(),
+            'error' => $exception?->getMessage(),
+            'trace' => $exception?->getTraceAsString(),
+        ]);
+
+        // Reload execution log from database
+        // When a job times out, failed() is called in a fresh process with the original
+        // queue payload, so $executionId will be null. We need to query for the latest execution.
+        $execution = null;
+
+        // Try to find execution using stored ID first (works for non-timeout failures)
+        if ($this->executionId) {
+            $execution = ScheduledTaskExecution::find($this->executionId);
+        }
+
+        // If no stored ID or not found, query for the most recent execution log for this task
+        if (! $execution) {
+            $execution = ScheduledTaskExecution::query()
+                ->where('scheduled_task_id', $this->task->id)
+                ->orderBy('created_at', 'desc')
+                ->first();
+        }
+
+        // Last resort: check task_log property
+        if (! $execution && $this->task_log) {
+            $execution = $this->task_log;
+        }
+
+        if ($execution) {
+            $errorMessage = 'Job permanently failed after '.$this->attempts().' attempts';
+            if ($exception) {
+                $errorMessage .= ': '.$exception->getMessage();
+            }
+
+            $execution->update([
+                'status' => 'failed',
+                'message' => $errorMessage,
+                'error_details' => $exception?->getTraceAsString(),
+                'finished_at' => Carbon::now()->toImmutable(),
+            ]);
+        } else {
+            Log::channel('scheduled-errors')->warning('Could not find execution log to update', [
+                'execution_id' => $this->executionId,
+                'task_id' => $this->task->uuid,
+            ]);
+        }
+
+        // Notify team about permanent failure
+        $this->team?->notify(new TaskFailed($this->task, $exception?->getMessage() ?? 'Unknown error'));
+    }
 }

app/Jobs/ServerManagerJob.php

@@ -87,7 +87,7 @@ class ServerManagerJob implements ShouldQueue
                     Log::channel('scheduled-errors')->error('Failed to dispatch ServerConnectionCheck', [
                         'server_id' => $server->id,
                         'server_name' => $server->name,
-                        'error' => $e->getMessage(),
+                        'error' => get_class($e).': '.$e->getMessage(),
                     ]);
                 }
             });
@@ -103,7 +103,7 @@ class ServerManagerJob implements ShouldQueue
                 Log::channel('scheduled-errors')->error('Error processing server tasks', [
                     'server_id' => $server->id,
                     'server_name' => $server->name,
-                    'error' => $e->getMessage(),
+                    'error' => get_class($e).': '.$e->getMessage(),
                 ]);
             }
         }

app/Jobs/ValidateAndInstallServerJob.php

@@ -72,6 +72,42 @@ class ValidateAndInstallServerJob implements ShouldQueue
                 return;
             }
 
+            // Check and install prerequisites
+            $validationResult = $this->server->validatePrerequisites();
+            if (! $validationResult['success']) {
+                if ($this->numberOfTries >= $this->maxTries) {
+                    $missingCommands = implode(', ', $validationResult['missing']);
+                    $errorMessage = "Prerequisites ({$missingCommands}) could not be installed after {$this->maxTries} attempts. Please install them manually before continuing.";
+                    $this->server->update([
+                        'validation_logs' => $errorMessage,
+                        'is_validating' => false,
+                    ]);
+                    Log::error('ValidateAndInstallServer: Prerequisites installation failed after max tries', [
+                        'server_id' => $this->server->id,
+                        'attempts' => $this->numberOfTries,
+                        'missing_commands' => $validationResult['missing'],
+                        'found_commands' => $validationResult['found'],
+                    ]);
+
+                    return;
+                }
+
+                Log::info('ValidateAndInstallServer: Installing prerequisites', [
+                    'server_id' => $this->server->id,
+                    'attempt' => $this->numberOfTries + 1,
+                    'missing_commands' => $validationResult['missing'],
+                    'found_commands' => $validationResult['found'],
+                ]);
+
+                // Install prerequisites
+                $this->server->installPrerequisites();
+
+                // Retry validation after installation
+                self::dispatch($this->server, $this->numberOfTries + 1)->delay(now()->addSeconds(30));
+
+                return;
+            }
+
             // Check if Docker is installed
             $dockerInstalled = $this->server->validateDockerEngine();
             $dockerComposeInstalled = $this->server->validateDockerCompose();
@@ -132,6 +168,9 @@ class ValidateAndInstallServerJob implements ShouldQueue
             if (! $this->server->isBuildServer()) {
                 $proxyShouldRun = CheckProxy::run($this->server, true);
                 if ($proxyShouldRun) {
+                    // Ensure networks exist BEFORE dispatching async proxy startup
+                    // This prevents race condition where proxy tries to start before networks are created
+                    instant_remote_process(ensureProxyNetworksExist($this->server)->toArray(), $this->server, false);
                     StartProxy::dispatch($this->server);
                 }
             }