Merge remote-tracking branch 'origin/next' into jean/port-exposes-improvement

app/Jobs/ApiTokenExpirationWarningJob.php

@@ -0,0 +1,64 @@
+<?php
+
+namespace App\Jobs;
+
+use App\Models\PersonalAccessToken;
+use App\Models\Team;
+use App\Models\User;
+use App\Notifications\ApiTokenExpiringNotification;
+use Illuminate\Bus\Queueable;
+use Illuminate\Contracts\Queue\ShouldBeEncrypted;
+use Illuminate\Contracts\Queue\ShouldQueue;
+use Illuminate\Foundation\Bus\Dispatchable;
+use Illuminate\Queue\InteractsWithQueue;
+use Illuminate\Queue\SerializesModels;
+use Laravel\Horizon\Contracts\Silenced;
+
+class ApiTokenExpirationWarningJob implements ShouldBeEncrypted, ShouldQueue, Silenced
+{
+    use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
+
+    public $tries = 1;
+
+    public $timeout = 120;
+
+    public function handle(): void
+    {
+        PersonalAccessToken::query()
+            ->whereNotNull('expires_at')
+            ->where('expires_at', '>', now())
+            ->where('expires_at', '<=', now()->addDay())
+            ->whereNull('api_token_expiration_warning_sent_at')
+            ->where('tokenable_type', User::class)
+            ->chunkById(100, function ($tokens) {
+                foreach ($tokens as $token) {
+                    if (! $token->team_id) {
+                        continue;
+                    }
+
+                    $team = Team::find($token->team_id);
+                    if (! $team) {
+                        continue;
+                    }
+
+                    $warningSentAt = now();
+
+                    $team->notify(new ApiTokenExpiringNotification($token));
+
+                    $markedAsSent = PersonalAccessToken::query()
+                        ->whereKey($token->getKey())
+                        ->whereNotNull('expires_at')
+                        ->where('expires_at', '>', now())
+                        ->where('expires_at', '<=', now()->addDay())
+                        ->whereNull('api_token_expiration_warning_sent_at')
+                        ->update(['api_token_expiration_warning_sent_at' => $warningSentAt]);
+
+                    if ($markedAsSent !== 1) {
+                        continue;
+                    }
+
+                    $token->forceFill(['api_token_expiration_warning_sent_at' => $warningSentAt]);
+                }
+            });
+    }
+}

app/Jobs/CleanupInstanceStuffsJob.php

@@ -2,6 +2,7 @@
 
 namespace App\Jobs;
 
+use App\Models\ScheduledDatabaseBackup;
 use App\Models\TeamInvitation;
 use App\Models\User;
 use Illuminate\Bus\Queueable;
@@ -12,6 +13,7 @@ use Illuminate\Foundation\Bus\Dispatchable;
 use Illuminate\Queue\InteractsWithQueue;
 use Illuminate\Queue\Middleware\WithoutOverlapping;
 use Illuminate\Queue\SerializesModels;
+use Illuminate\Support\Facades\Cache;
 use Illuminate\Support\Facades\Log;
 
 class CleanupInstanceStuffsJob implements ShouldBeEncrypted, ShouldBeUnique, ShouldQueue
@@ -32,6 +34,7 @@ class CleanupInstanceStuffsJob implements ShouldBeEncrypted, ShouldBeUnique, Sho
         try {
             $this->cleanupInvitationLink();
             $this->cleanupExpiredEmailChangeRequests();
+            $this->enforceBackupRetention();
         } catch (\Throwable $e) {
             Log::error('CleanupInstanceStuffsJob failed with error: '.$e->getMessage());
         }
@@ -55,4 +58,25 @@ class CleanupInstanceStuffsJob implements ShouldBeEncrypted, ShouldBeUnique, Sho
                 'email_change_code_expires_at' => null,
             ]);
     }
+
+    private function enforceBackupRetention(): void
+    {
+        if (! Cache::add('backup-retention-enforcement', true, 1800)) {
+            return;
+        }
+
+        try {
+            $backups = ScheduledDatabaseBackup::where('enabled', true)->get();
+            foreach ($backups as $backup) {
+                try {
+                    removeOldBackups($backup);
+                } catch (\Throwable $e) {
+                    Log::warning('Failed to enforce retention for backup '.$backup->id.': '.$e->getMessage());
+                }
+            }
+        } catch (\Throwable $e) {
+            Log::error('Failed to enforce backup retention: '.$e->getMessage());
+            Cache::forget('backup-retention-enforcement');
+        }
+    }
 }

app/Jobs/CleanupStaleMultiplexedConnections.php

@@ -9,6 +9,7 @@ use Illuminate\Contracts\Queue\ShouldQueue;
 use Illuminate\Foundation\Bus\Dispatchable;
 use Illuminate\Queue\InteractsWithQueue;
 use Illuminate\Queue\SerializesModels;
+use Illuminate\Support\Facades\Log;
 use Illuminate\Support\Facades\Process;
 use Illuminate\Support\Facades\Storage;
 
@@ -20,6 +21,132 @@ class CleanupStaleMultiplexedConnections implements ShouldQueue
     {
         $this->cleanupStaleConnections();
         $this->cleanupNonExistentServerConnections();
+        $this->cleanupOrphanedSshProcesses();
+        $this->cleanupOrphanedCloudflaredProcesses();
+    }
+
+    /**
+     * Kill backgrounded ssh master processes that lost the ControlPath socket
+     * race. Such processes are not masters, so ControlPersist never reaps them
+     * and they leak memory until the container restarts. A legitimate master
+     * always owns its socket file; an orphan has none.
+     *
+     * Processes younger than the minimum age are skipped: a freshly forked
+     * master creates its socket a few milliseconds after starting, so a young
+     * process with no socket may simply be mid-establish rather than orphaned.
+     */
+    private function cleanupOrphanedSshProcesses(): void
+    {
+        $muxDir = storage_path('app/ssh/mux');
+        $minAge = (int) config('constants.ssh.mux_orphan_min_age');
+
+        foreach ($this->listProcesses() as $process) {
+            // Backgrounded ssh master: current `ssh -fN` or legacy `ssh -fNM`.
+            if (! preg_match('#(^|/)ssh -fN#', $process['args'])) {
+                continue;
+            }
+
+            // Only ever touch ssh processes pointing at Coolify's mux directory.
+            if (! preg_match('#ControlPath=('.preg_quote($muxDir, '#').'/\S+)#', $process['args'], $pathMatch)) {
+                continue;
+            }
+
+            if ($process['etimes'] >= $minAge && ! file_exists($pathMatch[1])) {
+                $this->reapOrphan('ssh', $process);
+            }
+        }
+    }
+
+    /**
+     * Kill orphaned `cloudflared access ssh` proxy processes. Each is spawned
+     * as the SSH ProxyCommand transport for a Cloudflare Tunnel server and must
+     * die with its parent ssh. When that ssh is killed or orphaned (e.g. a lost
+     * mux master), the cloudflared process can leak and accumulate. A legitimate
+     * proxy always has a live ssh parent; one without is safe to reap.
+     *
+     * Processes younger than the minimum age are skipped so a proxy whose parent
+     * ssh is still starting up, or a transient `ssh -O check` proxy mid-exit, is
+     * never mistaken for an orphan.
+     */
+    private function cleanupOrphanedCloudflaredProcesses(): void
+    {
+        $minAge = (int) config('constants.ssh.mux_orphan_min_age');
+        $processes = $this->listProcesses();
+
+        $sshPids = [];
+        foreach ($processes as $process) {
+            // The ssh binary itself, not `cloudflared access ssh` (space before ssh).
+            if (preg_match('#(^|/)ssh\s#', $process['args'])) {
+                $sshPids[$process['pid']] = true;
+            }
+        }
+
+        foreach ($processes as $process) {
+            // `cloudflared access ssh`, never the `cloudflared tunnel` daemon.
+            if (! str_contains($process['args'], 'cloudflared access ssh')) {
+                continue;
+            }
+
+            // Orphaned when no live ssh process is its parent.
+            if ($process['etimes'] >= $minAge && ! isset($sshPids[$process['ppid']])) {
+                $this->reapOrphan('cloudflared', $process);
+            }
+        }
+    }
+
+    /**
+     * Reap a detected orphan process. When orphan reaping is disabled (the
+     * default), the orphan is only logged — a dry-run mode that lets operators
+     * verify what would be killed before enabling it for real.
+     *
+     * @param  array{pid: string, ppid: string, etimes: int, args: string}  $process
+     */
+    private function reapOrphan(string $kind, array $process): void
+    {
+        if (! config('constants.ssh.mux_orphan_reap_enabled')) {
+            Log::info("Orphaned {$kind} process detected (dry-run, not killed)", [
+                'pid' => $process['pid'],
+                'etimes' => $process['etimes'],
+                'command' => $process['args'],
+            ]);
+
+            return;
+        }
+
+        Process::run('kill '.escapeshellarg($process['pid']));
+        Log::info("Killed orphaned {$kind} process", [
+            'pid' => $process['pid'],
+            'etimes' => $process['etimes'],
+            'command' => $process['args'],
+        ]);
+    }
+
+    /**
+     * Snapshot of running processes.
+     *
+     * @return list<array{pid: string, ppid: string, etimes: int, args: string}>
+     */
+    private function listProcesses(): array
+    {
+        $ps = Process::run('ps -ww -eo pid=,ppid=,etimes=,args=');
+        if ($ps->exitCode() !== 0) {
+            return [];
+        }
+
+        $processes = [];
+        foreach (explode("\n", trim($ps->output())) as $line) {
+            if (! preg_match('/^\s*(\d+)\s+(\d+)\s+(\d+)\s+(.*)$/', $line, $matches)) {
+                continue;
+            }
+            $processes[] = [
+                'pid' => $matches[1],
+                'ppid' => $matches[2],
+                'etimes' => (int) $matches[3],
+                'args' => $matches[4],
+            ];
+        }
+
+        return $processes;
     }
 
     private function cleanupStaleConnections()
@@ -31,7 +158,7 @@ class CleanupStaleMultiplexedConnections implements ShouldQueue
             $server = Server::where('uuid', $serverUuid)->first();
 
             if (! $server) {
-                $this->removeMultiplexFile($muxFile);
+                $this->removeMultiplexFile($muxFile, 'server_not_found');
 
                 continue;
             }
@@ -41,14 +168,14 @@ class CleanupStaleMultiplexedConnections implements ShouldQueue
             $checkProcess = Process::run($checkCommand);
 
             if ($checkProcess->exitCode() !== 0) {
-                $this->removeMultiplexFile($muxFile);
+                $this->removeMultiplexFile($muxFile, 'connection_check_failed');
             } else {
                 $muxContent = Storage::disk('ssh-mux')->get($muxFile);
                 $establishedAt = Carbon::parse(substr($muxContent, 37));
                 $expirationTime = $establishedAt->addSeconds(config('constants.ssh.mux_persist_time'));
 
                 if (Carbon::now()->isAfter($expirationTime)) {
-                    $this->removeMultiplexFile($muxFile);
+                    $this->removeMultiplexFile($muxFile, 'expired');
                 }
             }
         }
@@ -62,7 +189,7 @@ class CleanupStaleMultiplexedConnections implements ShouldQueue
         foreach ($muxFiles as $muxFile) {
             $serverUuid = $this->extractServerUuidFromMuxFile($muxFile);
             if (! in_array($serverUuid, $existingServerUuids)) {
-                $this->removeMultiplexFile($muxFile);
+                $this->removeMultiplexFile($muxFile, 'server_does_not_exist');
             }
         }
     }
@@ -72,11 +199,30 @@ class CleanupStaleMultiplexedConnections implements ShouldQueue
         return substr($muxFile, 4);
     }
 
-    private function removeMultiplexFile($muxFile)
+    /**
+     * Close and delete a stale mux socket file. When orphan reaping is disabled
+     * (the default), the file is only logged — a dry-run mode that lets operators
+     * verify what would be removed before enabling it for real.
+     */
+    private function removeMultiplexFile(string $muxFile, string $reason): void
     {
+        if (! config('constants.ssh.mux_orphan_reap_enabled')) {
+            Log::info('Stale mux file detected (dry-run, not removed)', [
+                'file' => $muxFile,
+                'reason' => $reason,
+            ]);
+
+            return;
+        }
+
         $muxSocket = "/var/www/html/storage/app/ssh/mux/{$muxFile}";
         $closeCommand = "ssh -O exit -o ControlPath={$muxSocket} localhost 2>/dev/null";
         Process::run($closeCommand);
         Storage::disk('ssh-mux')->delete($muxFile);
+
+        Log::info('Removed stale mux file', [
+            'file' => $muxFile,
+            'reason' => $reason,
+        ]);
     }
 }

app/Jobs/DatabaseBackupJob.php

@@ -22,6 +22,7 @@ use Illuminate\Contracts\Queue\ShouldBeEncrypted;
 use Illuminate\Contracts\Queue\ShouldQueue;
 use Illuminate\Foundation\Bus\Dispatchable;
 use Illuminate\Queue\InteractsWithQueue;
+use Illuminate\Queue\Middleware\WithoutOverlapping;
 use Illuminate\Queue\SerializesModels;
 use Illuminate\Support\Facades\Log;
 use Illuminate\Support\Str;
@@ -76,10 +77,17 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue
 
     public function __construct(public ScheduledDatabaseBackup $backup)
     {
-        $this->onQueue('high');
+        $this->onQueue(crons_queue());
         $this->timeout = $backup->timeout ?? 3600;
     }
 
+    public function middleware(): array
+    {
+        $expireAfter = ($this->backup->timeout ?? 3600) + 300;
+
+        return [(new WithoutOverlapping('database-backup-'.$this->backup->id))->expireAfter($expireAfter)->dontRelease()];
+    }
+
     public function handle(): void
     {
         try {
@@ -91,7 +99,7 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue
 
                 return;
             }
-            if (data_get($this->backup, 'database_type') === \App\Models\ServiceDatabase::class) {
+            if (data_get($this->backup, 'database_type') === ServiceDatabase::class) {
                 $this->database = data_get($this->backup, 'database');
                 $this->server = $this->database->service->server;
                 $this->s3 = $this->backup->s3;
@@ -107,6 +115,8 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue
                 throw new \Exception('Database not found?!');
             }
 
+            $this->markStaleExecutionsAsFailed();
+
             BackupCreated::dispatch($this->team->id);
 
             $status = str(data_get($this->database, 'status'));
@@ -119,7 +129,7 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue
 
                 return;
             }
-            if (data_get($this->backup, 'database_type') === \App\Models\ServiceDatabase::class) {
+            if (data_get($this->backup, 'database_type') === ServiceDatabase::class) {
                 $databaseType = $this->database->databaseType();
                 $serviceUuid = $this->database->service->uuid;
                 $serviceName = str($this->database->service->name)->slug();
@@ -241,7 +251,7 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue
                             }
                         }
 
-                    } catch (\Throwable $e) {
+                    } catch (Throwable $e) {
                         // Continue without env vars - will be handled in backup_standalone_mongodb method
                     }
                 }
@@ -388,7 +398,7 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue
                     } else {
                         throw new \Exception('Local backup file is empty or was not created');
                     }
-                } catch (\Throwable $e) {
+                } catch (Throwable $e) {
                     // Local backup failed
                     if ($this->backup_log) {
                         $this->backup_log->update([
@@ -401,7 +411,7 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue
                     }
                     try {
                         $this->team?->notify(new BackupFailed($this->backup, $this->database, $this->error_output ?? $this->backup_output ?? $e->getMessage(), $database));
-                    } catch (\Throwable $notifyException) {
+                    } catch (Throwable $notifyException) {
                         Log::channel('scheduled-errors')->warning('Failed to send backup failure notification', [
                             'backup_id' => $this->backup->uuid,
                             'database' => $database,
@@ -423,7 +433,7 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue
                             deleteBackupsLocally($this->backup_location, $this->server);
                             $localStorageDeleted = true;
                         }
-                    } catch (\Throwable $e) {
+                    } catch (Throwable $e) {
                         // S3 upload failed but local backup succeeded
                         $s3UploadError = $e->getMessage();
                     }
@@ -455,7 +465,7 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue
                         } else {
                             $this->team->notify(new BackupSuccess($this->backup, $this->database, $database));
                         }
-                    } catch (\Throwable $e) {
+                    } catch (Throwable $e) {
                         Log::channel('scheduled-errors')->warning('Failed to send backup success notification', [
                             'backup_id' => $this->backup->uuid,
                             'database' => $database,
@@ -467,7 +477,7 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue
             if ($this->backup_log && $this->backup_log->status === 'success') {
                 removeOldBackups($this->backup);
             }
-        } catch (\Throwable $e) {
+        } catch (Throwable $e) {
             throw $e;
         } finally {
             if ($this->team) {
@@ -489,19 +499,23 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue
                 // For service-based MongoDB, try to build URL from environment variables
                 if (filled($this->mongo_root_username) && filled($this->mongo_root_password)) {
                     // Use container name instead of server IP for service-based MongoDB
-                    $url = "mongodb://{$this->mongo_root_username}:{$this->mongo_root_password}@{$this->container_name}:27017";
+                    // URL-encode credentials to prevent URI injection
+                    $encodedUser = rawurlencode($this->mongo_root_username);
+                    $encodedPass = rawurlencode($this->mongo_root_password);
+                    $url = "mongodb://{$encodedUser}:{$encodedPass}@{$this->container_name}:27017";
                 } else {
                     // If no environment variables are available, throw an exception
                     throw new \Exception('MongoDB credentials not found. Ensure MONGO_INITDB_ROOT_USERNAME and MONGO_INITDB_ROOT_PASSWORD environment variables are available in the container.');
                 }
             }
             Log::info('MongoDB backup URL configured', ['has_url' => filled($url), 'using_env_vars' => blank($this->database->internal_db_url)]);
+            $escapedUrl = escapeshellarg($url);
             if ($databaseWithCollections === 'all') {
                 $commands[] = 'mkdir -p '.$this->backup_dir;
                 if (str($this->database->image)->startsWith('mongo:4')) {
-                    $commands[] = "docker exec $this->container_name mongodump --uri=\"$url\" --gzip --archive > $this->backup_location";
+                    $commands[] = "docker exec $this->container_name mongodump --uri=$escapedUrl --gzip --archive > $this->backup_location";
                 } else {
-                    $commands[] = "docker exec $this->container_name mongodump --authenticationDatabase=admin --uri=\"$url\" --gzip --archive > $this->backup_location";
+                    $commands[] = "docker exec $this->container_name mongodump --authenticationDatabase=admin --uri=$escapedUrl --gzip --archive > $this->backup_location";
                 }
             } else {
                 if (str($databaseWithCollections)->contains(':')) {
@@ -519,9 +533,9 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue
 
                 if ($collectionsToExclude->count() === 0) {
                     if (str($this->database->image)->startsWith('mongo:4')) {
-                        $commands[] = "docker exec $this->container_name mongodump --uri=\"$url\" --gzip --archive > $this->backup_location";
+                        $commands[] = "docker exec $this->container_name mongodump --uri=$escapedUrl --gzip --archive > $this->backup_location";
                     } else {
-                        $commands[] = "docker exec $this->container_name mongodump --authenticationDatabase=admin --uri=\"$url\" --db $escapedDatabaseName --gzip --archive > $this->backup_location";
+                        $commands[] = "docker exec $this->container_name mongodump --authenticationDatabase=admin --uri=$escapedUrl --db $escapedDatabaseName --gzip --archive > $this->backup_location";
                     }
                 } else {
                     // Validate and escape each collection name
@@ -533,9 +547,9 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue
                     });
 
                     if (str($this->database->image)->startsWith('mongo:4')) {
-                        $commands[] = "docker exec $this->container_name mongodump --uri=$url --gzip --excludeCollection ".$escapedCollections->implode(' --excludeCollection ')." --archive > $this->backup_location";
+                        $commands[] = "docker exec $this->container_name mongodump --uri=$escapedUrl --gzip --excludeCollection ".$escapedCollections->implode(' --excludeCollection ')." --archive > $this->backup_location";
                     } else {
-                        $commands[] = "docker exec $this->container_name mongodump --authenticationDatabase=admin --uri=\"$url\" --db $escapedDatabaseName --gzip --excludeCollection ".$escapedCollections->implode(' --excludeCollection ')." --archive > $this->backup_location";
+                        $commands[] = "docker exec $this->container_name mongodump --authenticationDatabase=admin --uri=$escapedUrl --db $escapedDatabaseName --gzip --excludeCollection ".$escapedCollections->implode(' --excludeCollection ')." --archive > $this->backup_location";
                     }
                 }
             }
@@ -544,7 +558,7 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue
             if ($this->backup_output === '') {
                 $this->backup_output = null;
             }
-        } catch (\Throwable $e) {
+        } catch (Throwable $e) {
             $this->add_to_error_output($e->getMessage());
             throw $e;
         }
@@ -556,15 +570,16 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue
             $commands[] = 'mkdir -p '.$this->backup_dir;
             $backupCommand = 'docker exec';
             if ($this->postgres_password) {
-                $backupCommand .= " -e PGPASSWORD=\"{$this->postgres_password}\"";
+                $backupCommand .= ' -e PGPASSWORD='.escapeshellarg($this->postgres_password);
             }
+            $escapedUsername = escapeshellarg($this->database->postgres_user);
             if ($this->backup->dump_all) {
-                $backupCommand .= " $this->container_name pg_dumpall --username {$this->database->postgres_user} | gzip > $this->backup_location";
+                $backupCommand .= " $this->container_name pg_dumpall --username $escapedUsername | gzip > $this->backup_location";
             } else {
                 // Validate and escape database name to prevent command injection
                 validateShellSafePath($database, 'database name');
                 $escapedDatabase = escapeshellarg($database);
-                $backupCommand .= " $this->container_name pg_dump --format=custom --no-acl --no-owner --username {$this->database->postgres_user} $escapedDatabase > $this->backup_location";
+                $backupCommand .= " $this->container_name pg_dump --format=custom --no-acl --no-owner --username $escapedUsername $escapedDatabase > $this->backup_location";
             }
 
             $commands[] = $backupCommand;
@@ -573,7 +588,7 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue
             if ($this->backup_output === '') {
                 $this->backup_output = null;
             }
-        } catch (\Throwable $e) {
+        } catch (Throwable $e) {
             $this->add_to_error_output($e->getMessage());
             throw $e;
         }
@@ -583,20 +598,21 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue
     {
         try {
             $commands[] = 'mkdir -p '.$this->backup_dir;
+            $escapedPassword = escapeshellarg($this->database->mysql_root_password);
             if ($this->backup->dump_all) {
-                $commands[] = "docker exec $this->container_name mysqldump -u root -p\"{$this->database->mysql_root_password}\" --all-databases --single-transaction --quick --lock-tables=false --compress | gzip > $this->backup_location";
+                $commands[] = "docker exec $this->container_name mysqldump -u root -p$escapedPassword --all-databases --single-transaction --quick --lock-tables=false --compress | gzip > $this->backup_location";
             } else {
                 // Validate and escape database name to prevent command injection
                 validateShellSafePath($database, 'database name');
                 $escapedDatabase = escapeshellarg($database);
-                $commands[] = "docker exec $this->container_name mysqldump -u root -p\"{$this->database->mysql_root_password}\" $escapedDatabase > $this->backup_location";
+                $commands[] = "docker exec $this->container_name mysqldump -u root -p$escapedPassword $escapedDatabase > $this->backup_location";
             }
             $this->backup_output = instant_remote_process($commands, $this->server, true, false, $this->timeout, disableMultiplexing: true);
             $this->backup_output = trim($this->backup_output);
             if ($this->backup_output === '') {
                 $this->backup_output = null;
             }
-        } catch (\Throwable $e) {
+        } catch (Throwable $e) {
             $this->add_to_error_output($e->getMessage());
             throw $e;
         }
@@ -606,20 +622,21 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue
     {
         try {
             $commands[] = 'mkdir -p '.$this->backup_dir;
+            $escapedPassword = escapeshellarg($this->database->mariadb_root_password);
             if ($this->backup->dump_all) {
-                $commands[] = "docker exec $this->container_name mariadb-dump -u root -p\"{$this->database->mariadb_root_password}\" --all-databases --single-transaction --quick --lock-tables=false --compress > $this->backup_location";
+                $commands[] = "docker exec $this->container_name mariadb-dump -u root -p$escapedPassword --all-databases --single-transaction --quick --lock-tables=false --compress > $this->backup_location";
             } else {
                 // Validate and escape database name to prevent command injection
                 validateShellSafePath($database, 'database name');
                 $escapedDatabase = escapeshellarg($database);
-                $commands[] = "docker exec $this->container_name mariadb-dump -u root -p\"{$this->database->mariadb_root_password}\" $escapedDatabase > $this->backup_location";
+                $commands[] = "docker exec $this->container_name mariadb-dump -u root -p$escapedPassword $escapedDatabase > $this->backup_location";
             }
             $this->backup_output = instant_remote_process($commands, $this->server, true, false, $this->timeout, disableMultiplexing: true);
             $this->backup_output = trim($this->backup_output);
             if ($this->backup_output === '') {
                 $this->backup_output = null;
             }
-        } catch (\Throwable $e) {
+        } catch (Throwable $e) {
             $this->add_to_error_output($e->getMessage());
             throw $e;
         }
@@ -651,12 +668,14 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue
     private function upload_to_s3(): void
     {
         if (is_null($this->s3)) {
+            $previousS3StorageId = $this->backup->s3_storage_id;
+
             $this->backup->update([
                 'save_s3' => false,
                 's3_storage_id' => null,
             ]);
 
-            throw new \Exception('S3 storage configuration is missing or has been deleted (S3 storage ID: '.($this->backup->s3_storage_id ?? 'null').'). S3 backup has been disabled for this schedule.');
+            throw new \Exception('S3 storage configuration is missing or has been deleted (S3 storage ID: '.($previousS3StorageId ?? 'null').'). S3 backup has been disabled for this schedule.');
         }
 
         try {
@@ -666,11 +685,12 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue
             $bucket = $this->s3->bucket;
             $endpoint = $this->s3->endpoint;
             $this->s3->testConnection(shouldSave: true);
-            if (data_get($this->backup, 'database_type') === \App\Models\ServiceDatabase::class) {
+            if (data_get($this->backup, 'database_type') === ServiceDatabase::class) {
                 $network = $this->database->service->destination->network;
             } else {
                 $network = $this->database->destination->network;
             }
+            $safeNetwork = escapeshellarg($network);
 
             $fullImageName = $this->getFullImageName();
 
@@ -682,13 +702,13 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue
             if (isDev()) {
                 if ($this->database->name === 'coolify-db') {
                     $backup_location_from = '/var/lib/docker/volumes/coolify_dev_backups_data/_data/coolify/coolify-db-'.$this->server->ip.$this->backup_file;
-                    $commands[] = "docker run -d --network {$network} --name backup-of-{$this->backup_log_uuid} --rm -v $backup_location_from:$this->backup_location:ro {$fullImageName}";
+                    $commands[] = "docker run -d --network {$safeNetwork} --name backup-of-{$this->backup_log_uuid} --rm -v $backup_location_from:$this->backup_location:ro {$fullImageName}";
                 } else {
                     $backup_location_from = '/var/lib/docker/volumes/coolify_dev_backups_data/_data/databases/'.str($this->team->name)->slug().'-'.$this->team->id.'/'.$this->directory_name.$this->backup_file;
-                    $commands[] = "docker run -d --network {$network} --name backup-of-{$this->backup_log_uuid} --rm -v $backup_location_from:$this->backup_location:ro {$fullImageName}";
+                    $commands[] = "docker run -d --network {$safeNetwork} --name backup-of-{$this->backup_log_uuid} --rm -v $backup_location_from:$this->backup_location:ro {$fullImageName}";
                 }
             } else {
-                $commands[] = "docker run -d --network {$network} --name backup-of-{$this->backup_log_uuid} --rm -v $this->backup_location:$this->backup_location:ro {$fullImageName}";
+                $commands[] = "docker run -d --network {$safeNetwork} --name backup-of-{$this->backup_log_uuid} --rm -v $this->backup_location:$this->backup_location:ro {$fullImageName}";
             }
 
             // Escape S3 credentials to prevent command injection
@@ -701,7 +721,7 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue
             instant_remote_process($commands, $this->server, true, false, null, disableMultiplexing: true);
 
             $this->s3_uploaded = true;
-        } catch (\Throwable $e) {
+        } catch (Throwable $e) {
             $this->s3_uploaded = false;
             $this->add_to_error_output($e->getMessage());
             throw $e;
@@ -719,6 +739,31 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue
         return "{$helperImage}:{$latestVersion}";
     }
 
+    private function markStaleExecutionsAsFailed(): void
+    {
+        try {
+            $timeoutSeconds = ($this->backup->timeout ?? 3600) * 2;
+
+            $staleExecutions = $this->backup->executions()
+                ->where('status', 'running')
+                ->where('created_at', '<', now()->subSeconds($timeoutSeconds))
+                ->get();
+
+            foreach ($staleExecutions as $execution) {
+                $execution->update([
+                    'status' => 'failed',
+                    'message' => 'Marked as failed - backup execution exceeded maximum allowed time',
+                    'finished_at' => now(),
+                ]);
+            }
+        } catch (Throwable $e) {
+            Log::channel('scheduled-errors')->warning('Failed to clean up stale backup executions', [
+                'backup_id' => $this->backup->uuid,
+                'error' => $e->getMessage(),
+            ]);
+        }
+    }
+
     public function failed(?Throwable $exception): void
     {
         Log::channel('scheduled-errors')->error('DatabaseBackup permanently failed', [
@@ -755,7 +800,7 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue
             $output = $this->backup_output ?? $exception?->getMessage() ?? 'Unknown error';
             try {
                 $this->team->notify(new BackupFailed($this->backup, $this->database, $output, $databaseName));
-            } catch (\Throwable $e) {
+            } catch (Throwable $e) {
                 Log::channel('scheduled-errors')->warning('Failed to send backup permanent failure notification', [
                     'backup_id' => $this->backup->uuid,
                     'error' => $e->getMessage(),

app/Jobs/ProcessGithubPullRequestWebhook.php

@@ -4,6 +4,7 @@ namespace App\Jobs;
 
 use App\Actions\Application\CleanupPreviewDeployment;
 use App\Enums\ProcessStatus;
+use App\Http\Controllers\Webhook\Concerns\DetectsSkipDeployCommits;
 use App\Models\Application;
 use App\Models\ApplicationPreview;
 use App\Models\GithubApp;
@@ -17,6 +18,7 @@ use Visus\Cuid2\Cuid2;
 
 class ProcessGithubPullRequestWebhook implements ShouldBeEncrypted, ShouldQueue
 {
+    use DetectsSkipDeployCommits;
     use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
 
     public int $tries = 3;
@@ -31,11 +33,13 @@ class ProcessGithubPullRequestWebhook implements ShouldBeEncrypted, ShouldQueue
         public string $action,
         public int $pullRequestId,
         public string $pullRequestHtmlUrl,
+        public ?string $pullRequestTitle,
         public ?string $beforeSha,
         public ?string $afterSha,
         public string $commitSha,
         public ?string $authorAssociation,
         public string $fullName,
+        public bool $isForkPullRequest = false,
     ) {
         $this->onQueue('high');
     }
@@ -83,9 +87,23 @@ class ProcessGithubPullRequestWebhook implements ShouldBeEncrypted, ShouldQueue
             return;
         }
 
+        if (self::shouldSkipDeployAny([$this->pullRequestTitle])) {
+            return;
+        }
+
         // Check if PR deployments from public contributors are restricted
         if (! $application->settings->is_pr_deployments_public_enabled) {
-            $trustedAssociations = ['OWNER', 'MEMBER', 'COLLABORATOR', 'CONTRIBUTOR'];
+            // Fork PRs carry untrusted code from a repository outside our control.
+            // GitHub's author_association cannot be trusted to gate these (it grants
+            // CONTRIBUTOR to anyone who has merely opened an issue/PR before), so fork
+            // PRs are never deployed automatically when public previews are off.
+            if ($this->isForkPullRequest) {
+                return;
+            }
+
+            // Same-repo (non-fork) branch PRs require push access to the base repo,
+            // so only trusted associations are allowed to trigger a deployment.
+            $trustedAssociations = ['OWNER', 'MEMBER', 'COLLABORATOR'];
             if (! in_array($this->authorAssociation, $trustedAssociations)) {
                 return;
             }

app/Jobs/PushServerUpdateJob.php

@@ -13,6 +13,16 @@ use App\Models\ApplicationPreview;
 use App\Models\Server;
 use App\Models\ServiceApplication;
 use App\Models\ServiceDatabase;
+use App\Models\StandaloneClickhouse;
+use App\Models\StandaloneDocker;
+use App\Models\StandaloneDragonfly;
+use App\Models\StandaloneKeydb;
+use App\Models\StandaloneMariadb;
+use App\Models\StandaloneMongodb;
+use App\Models\StandaloneMysql;
+use App\Models\StandalonePostgresql;
+use App\Models\StandaloneRedis;
+use App\Models\SwarmDocker;
 use App\Notifications\Container\ContainerRestarted;
 use App\Services\ContainerStatusAggregator;
 use App\Traits\CalculatesExcludedStatus;
@@ -25,6 +35,7 @@ use Illuminate\Queue\Middleware\WithoutOverlapping;
 use Illuminate\Queue\SerializesModels;
 use Illuminate\Support\Collection;
 use Illuminate\Support\Facades\Cache;
+use Illuminate\Support\Facades\DB;
 use Laravel\Horizon\Contracts\Silenced;
 
 class PushServerUpdateJob implements ShouldBeEncrypted, ShouldQueue, Silenced
@@ -46,6 +57,18 @@ class PushServerUpdateJob implements ShouldBeEncrypted, ShouldQueue, Silenced
 
     public Collection $services;
 
+    public Collection $applicationsById;
+
+    public Collection $previewsByKey;
+
+    public Collection $databasesByUuid;
+
+    public Collection $servicesById;
+
+    public Collection $serviceApplicationsById;
+
+    public Collection $serviceDatabasesById;
+
     public Collection $allApplicationIds;
 
     public Collection $allDatabaseUuids;
@@ -78,6 +101,8 @@ class PushServerUpdateJob implements ShouldBeEncrypted, ShouldQueue, Silenced
 
     public bool $foundLogDrainContainer = false;
 
+    private ?array $cachedDestinationIds = null;
+
     public function middleware(): array
     {
         return [(new WithoutOverlapping('push-server-update-'.$this->server->uuid))->expireAfter(30)->dontRelease()];
@@ -103,6 +128,12 @@ class PushServerUpdateJob implements ShouldBeEncrypted, ShouldQueue, Silenced
         $this->allTcpProxyUuids = collect();
         $this->allServiceApplicationIds = collect();
         $this->allServiceDatabaseIds = collect();
+        $this->applicationsById = collect();
+        $this->previewsByKey = collect();
+        $this->databasesByUuid = collect();
+        $this->servicesById = collect();
+        $this->serviceApplicationsById = collect();
+        $this->serviceDatabasesById = collect();
     }
 
     public function handle()
@@ -120,6 +151,16 @@ class PushServerUpdateJob implements ShouldBeEncrypted, ShouldQueue, Silenced
         $this->allTcpProxyUuids ??= collect();
         $this->allServiceApplicationIds ??= collect();
         $this->allServiceDatabaseIds ??= collect();
+        $this->applicationsById ??= collect();
+        $this->previewsByKey ??= collect();
+        $this->databasesByUuid ??= collect();
+        $this->servicesById ??= collect();
+        $this->serviceApplicationsById ??= collect();
+        $this->serviceDatabasesById ??= collect();
+
+        // Eager-load relations the job touches repeatedly to avoid lazy-load queries
+        // (settings: disk threshold, isProxyShouldRun, isLogDrainEnabled; team: notifications).
+        $this->server->loadMissing(['settings', 'team']);
 
         // TODO: Swarm is not supported yet
         if (! $this->data) {
@@ -127,30 +168,40 @@ class PushServerUpdateJob implements ShouldBeEncrypted, ShouldQueue, Silenced
         }
         $data = collect($this->data);
 
-        $this->server->sentinelHeartbeat();
-
+        // Heartbeat is updated by SentinelController on every push, before dispatch.
         $this->containers = collect(data_get($data, 'containers'));
         $filesystemUsageRoot = data_get($data, 'filesystem_usage_root.used_percentage');
 
-        // Only dispatch storage check when disk percentage actually changes
+        // Only dispatch the storage check when disk usage is at/above the notification
+        // threshold AND the value changed. Below the threshold ServerStorageCheckJob
+        // has nothing to do (it only sends a HighDiskUsage notification), so dispatching
+        // it is wasted work — and most servers sit well below the threshold.
+        $diskThreshold = data_get($this->server, 'settings.server_disk_usage_notification_threshold', 80);
         $storageCacheKey = 'storage-check:'.$this->server->id;
         $lastPercentage = Cache::get($storageCacheKey);
-        if ($lastPercentage === null || (string) $lastPercentage !== (string) $filesystemUsageRoot) {
+        if ($filesystemUsageRoot !== null
+            && $filesystemUsageRoot >= $diskThreshold
+            && (string) $lastPercentage !== (string) $filesystemUsageRoot) {
             Cache::put($storageCacheKey, $filesystemUsageRoot, 600);
             ServerStorageCheckJob::dispatch($this->server, $filesystemUsageRoot);
+        } elseif ($filesystemUsageRoot !== null && $filesystemUsageRoot < $diskThreshold) {
+            Cache::forget($storageCacheKey);
         }
 
         if ($this->containers->isEmpty()) {
             return;
         }
 
-        $this->applications = $this->server->applications();
-        $this->databases = $this->server->databases();
-        $this->previews = $this->server->previews();
-        // Eager load service applications and databases to avoid N+1 queries
-        $this->services = $this->server->services()
-            ->with(['applications:id,service_id', 'databases:id,service_id'])
-            ->get();
+        $this->applications = $this->loadApplications();
+        $this->databases = $this->loadDatabases();
+        $this->previews = $this->loadPreviews();
+        $this->services = $this->loadServices();
+        $this->applicationsById = $this->applications->keyBy(fn ($application) => (string) $application->id);
+        $this->previewsByKey = $this->previews->keyBy(fn ($preview) => $preview->application_id.':'.$preview->pull_request_id);
+        $this->databasesByUuid = $this->databases->keyBy('uuid');
+        $this->servicesById = $this->services->keyBy(fn ($service) => (string) $service->id);
+        $this->serviceApplicationsById = $this->services->flatMap(fn ($service) => $service->applications)->keyBy(fn ($application) => (string) $application->id);
+        $this->serviceDatabasesById = $this->services->flatMap(fn ($service) => $service->databases)->keyBy(fn ($database) => (string) $database->id);
 
         $this->allApplicationIds = $this->applications->filter(function ($application) {
             return $application->additional_servers_count === 0;
@@ -163,9 +214,8 @@ class PushServerUpdateJob implements ShouldBeEncrypted, ShouldQueue, Silenced
         });
         $this->allDatabaseUuids = $this->databases->pluck('uuid');
         $this->allTcpProxyUuids = $this->databases->where('is_public', true)->pluck('uuid');
-        // Use eager-loaded relationships instead of querying in loop
-        $this->allServiceApplicationIds = $this->services->flatMap(fn ($service) => $service->applications->pluck('id'));
-        $this->allServiceDatabaseIds = $this->services->flatMap(fn ($service) => $service->databases->pluck('id'));
+        $this->allServiceApplicationIds = $this->serviceApplicationsById->keys();
+        $this->allServiceDatabaseIds = $this->serviceDatabasesById->keys();
 
         foreach ($this->containers as $container) {
             $containerStatus = data_get($container, 'state', 'exited');
@@ -279,6 +329,151 @@ class PushServerUpdateJob implements ShouldBeEncrypted, ShouldQueue, Silenced
         $this->checkLogDrainContainer();
     }
 
+    private function loadApplications(): Collection
+    {
+        [$standaloneDockerIds, $swarmDockerIds] = $this->serverDestinationIds();
+
+        $applications = ($standaloneDockerIds->isNotEmpty() || $swarmDockerIds->isNotEmpty())
+            ? Application::withoutGlobalScope('withRelations')
+                ->select([
+                    'id',
+                    'uuid',
+                    'name',
+                    'status',
+                    'build_pack',
+                    'docker_compose_raw',
+                    'destination_id',
+                    'destination_type',
+                    'last_online_at',
+                ])
+                ->withCount('additional_servers')
+                ->where(fn ($query) => $this->scopeDestination($query, $standaloneDockerIds, $swarmDockerIds))
+                ->get()
+            : collect();
+
+        $additionalApplicationIds = DB::table('additional_destinations')
+            ->where('server_id', $this->server->id)
+            ->pluck('application_id');
+
+        if ($additionalApplicationIds->isNotEmpty()) {
+            $applications = $applications->concat(
+                Application::withoutGlobalScope('withRelations')
+                    ->select([
+                        'id',
+                        'uuid',
+                        'name',
+                        'status',
+                        'build_pack',
+                        'docker_compose_raw',
+                        'destination_id',
+                        'destination_type',
+                        'last_online_at',
+                    ])
+                    ->withCount('additional_servers')
+                    ->whereIn('id', $additionalApplicationIds)
+                    ->get()
+            );
+        }
+
+        return $applications->unique('id')->values();
+    }
+
+    private function loadPreviews(): Collection
+    {
+        $applicationIds = $this->applications->pluck('id');
+
+        if ($applicationIds->isEmpty()) {
+            return collect();
+        }
+
+        return ApplicationPreview::query()
+            ->select([
+                'id',
+                'application_id',
+                'pull_request_id',
+                'status',
+                'last_online_at',
+            ])
+            ->whereIn('application_id', $applicationIds)
+            ->get();
+    }
+
+    private function loadServices(): Collection
+    {
+        return $this->server->services()
+            ->select([
+                'id',
+                'server_id',
+                'uuid',
+                'docker_compose_raw',
+            ])
+            ->with([
+                'applications:id,service_id,status,last_online_at',
+                'databases:id,service_id,status,last_online_at,is_public,name',
+            ])
+            ->get();
+    }
+
+    private function loadDatabases(): Collection
+    {
+        [$standaloneDockerIds, $swarmDockerIds] = $this->serverDestinationIds();
+        if ($standaloneDockerIds->isEmpty() && $swarmDockerIds->isEmpty()) {
+            return collect();
+        }
+        $databaseColumns = [
+            'id',
+            'uuid',
+            'name',
+            'status',
+            'is_public',
+            'destination_id',
+            'destination_type',
+            'last_online_at',
+            'restart_count',
+            'last_restart_at',
+            'last_restart_type',
+        ];
+
+        return collect([
+            StandalonePostgresql::class,
+            StandaloneRedis::class,
+            StandaloneMongodb::class,
+            StandaloneMysql::class,
+            StandaloneMariadb::class,
+            StandaloneKeydb::class,
+            StandaloneDragonfly::class,
+            StandaloneClickhouse::class,
+        ])->flatMap(function (string $databaseClass) use ($databaseColumns, $standaloneDockerIds, $swarmDockerIds) {
+            return $databaseClass::query()
+                ->select($databaseColumns)
+                ->where(fn ($query) => $this->scopeDestination($query, $standaloneDockerIds, $swarmDockerIds))
+                ->get();
+        })->filter(fn ($database) => data_get($database, 'name') !== 'coolify-db')->values();
+    }
+
+    private function serverDestinationIds(): array
+    {
+        if ($this->cachedDestinationIds !== null) {
+            return $this->cachedDestinationIds;
+        }
+
+        return $this->cachedDestinationIds = [
+            StandaloneDocker::where('server_id', $this->server->id)->pluck('id'),
+            SwarmDocker::where('server_id', $this->server->id)->pluck('id'),
+        ];
+    }
+
+    private function scopeDestination($query, Collection $standaloneDockerIds, Collection $swarmDockerIds): void
+    {
+        $query->where(function ($query) use ($standaloneDockerIds) {
+            $query->where('destination_type', StandaloneDocker::class)
+                ->whereIn('destination_id', $standaloneDockerIds);
+        })->orWhere(function ($query) use ($swarmDockerIds) {
+            $query->where('destination_type', SwarmDocker::class)
+                ->whereIn('destination_id', $swarmDockerIds);
+        });
+    }
+
     private function aggregateMultiContainerStatuses()
     {
         if ($this->applicationContainerStatuses->isEmpty()) {
@@ -286,7 +481,7 @@ class PushServerUpdateJob implements ShouldBeEncrypted, ShouldQueue, Silenced
         }
 
         foreach ($this->applicationContainerStatuses as $applicationId => $containerStatuses) {
-            $application = $this->applications->where('id', $applicationId)->first();
+            $application = $this->applicationsById->get((string) $applicationId);
             if (! $application) {
                 continue;
             }
@@ -307,8 +502,6 @@ class PushServerUpdateJob implements ShouldBeEncrypted, ShouldQueue, Silenced
                 if ($aggregatedStatus && $application->status !== $aggregatedStatus) {
                     $application->status = $aggregatedStatus;
                     $application->save();
-                } elseif ($aggregatedStatus) {
-                    $application->update(['last_online_at' => now()]);
                 }
 
                 continue;
@@ -323,8 +516,6 @@ class PushServerUpdateJob implements ShouldBeEncrypted, ShouldQueue, Silenced
             if ($aggregatedStatus && $application->status !== $aggregatedStatus) {
                 $application->status = $aggregatedStatus;
                 $application->save();
-            } elseif ($aggregatedStatus) {
-                $application->update(['last_online_at' => now()]);
             }
         }
     }
@@ -343,7 +534,7 @@ class PushServerUpdateJob implements ShouldBeEncrypted, ShouldQueue, Silenced
                 continue;
             }
 
-            $service = $this->services->where('id', $serviceId)->first();
+            $service = $this->servicesById->get((string) $serviceId);
             if (! $service) {
                 continue;
             }
@@ -351,9 +542,9 @@ class PushServerUpdateJob implements ShouldBeEncrypted, ShouldQueue, Silenced
             // Get the service sub-resource (ServiceApplication or ServiceDatabase)
             $subResource = null;
             if ($subType === 'application') {
-                $subResource = $service->applications->where('id', $subId)->first();
+                $subResource = $this->serviceApplicationsById->get((string) $subId);
             } elseif ($subType === 'database') {
-                $subResource = $service->databases->where('id', $subId)->first();
+                $subResource = $this->serviceDatabasesById->get((string) $subId);
             }
 
             if (! $subResource) {
@@ -375,8 +566,6 @@ class PushServerUpdateJob implements ShouldBeEncrypted, ShouldQueue, Silenced
                 if ($aggregatedStatus && $subResource->status !== $aggregatedStatus) {
                     $subResource->status = $aggregatedStatus;
                     $subResource->save();
-                } elseif ($aggregatedStatus) {
-                    $subResource->update(['last_online_at' => now()]);
                 }
 
                 continue;
@@ -392,39 +581,31 @@ class PushServerUpdateJob implements ShouldBeEncrypted, ShouldQueue, Silenced
             if ($aggregatedStatus && $subResource->status !== $aggregatedStatus) {
                 $subResource->status = $aggregatedStatus;
                 $subResource->save();
-            } elseif ($aggregatedStatus) {
-                $subResource->update(['last_online_at' => now()]);
             }
         }
     }
 
     private function updateApplicationStatus(string $applicationId, string $containerStatus)
     {
-        $application = $this->applications->where('id', $applicationId)->first();
+        $application = $this->applicationsById->get((string) $applicationId);
         if (! $application) {
             return;
         }
         if ($application->status !== $containerStatus) {
             $application->status = $containerStatus;
             $application->save();
-        } else {
-            $application->update(['last_online_at' => now()]);
         }
     }
 
     private function updateApplicationPreviewStatus(string $applicationId, string $pullRequestId, string $containerStatus)
     {
-        $application = $this->previews->where('application_id', $applicationId)
-            ->where('pull_request_id', $pullRequestId)
-            ->first();
+        $application = $this->previewsByKey->get($applicationId.':'.$pullRequestId);
         if (! $application) {
             return;
         }
         if ($application->status !== $containerStatus) {
             $application->status = $containerStatus;
             $application->save();
-        } else {
-            $application->update(['last_online_at' => now()]);
         }
     }
 
@@ -472,9 +653,7 @@ class PushServerUpdateJob implements ShouldBeEncrypted, ShouldQueue, Silenced
             $applicationId = $parts[0];
             $pullRequestId = $parts[1];
 
-            $applicationPreview = $this->previews->where('application_id', $applicationId)
-                ->where('pull_request_id', $pullRequestId)
-                ->first();
+            $applicationPreview = $this->previewsByKey->get($applicationId.':'.$pullRequestId);
 
             if ($applicationPreview && ! str($applicationPreview->status)->startsWith('exited')) {
                 $previewIdsToUpdate->push($applicationPreview->id);
@@ -500,11 +679,11 @@ class PushServerUpdateJob implements ShouldBeEncrypted, ShouldQueue, Silenced
                 } catch (\Throwable $e) {
                 }
             } else {
-                // Connect proxy to networks periodically (every 10 min) to avoid excessive job dispatches.
+                // Connect proxy to networks periodically as a safety net to avoid excessive job dispatches.
                 // On-demand triggers (new network, service deploy) use dispatchSync() and bypass this.
                 $proxyCacheKey = 'connect-proxy:'.$this->server->id;
                 if (! Cache::has($proxyCacheKey)) {
-                    Cache::put($proxyCacheKey, true, 600);
+                    Cache::put($proxyCacheKey, true, config('constants.proxy.connect_networks_interval_seconds', 3600));
                     ConnectProxyToNetworksJob::dispatch($this->server);
                 }
             }
@@ -513,15 +692,13 @@ class PushServerUpdateJob implements ShouldBeEncrypted, ShouldQueue, Silenced
 
     private function updateDatabaseStatus(string $databaseUuid, string $containerStatus, bool $tcpProxy = false)
     {
-        $database = $this->databases->where('uuid', $databaseUuid)->first();
+        $database = $this->databasesByUuid->get($databaseUuid);
         if (! $database) {
             return;
         }
         if ($database->status !== $containerStatus) {
             $database->status = $containerStatus;
             $database->save();
-        } else {
-            $database->update(['last_online_at' => now()]);
         }
         if ($this->isRunning($containerStatus) && $tcpProxy) {
             $tcpProxyContainerFound = $this->containers->filter(function ($value, $key) use ($databaseUuid) {
@@ -556,7 +733,7 @@ class PushServerUpdateJob implements ShouldBeEncrypted, ShouldQueue, Silenced
         }
 
         $notFoundDatabaseUuids->each(function ($databaseUuid) {
-            $database = $this->databases->where('uuid', $databaseUuid)->first();
+            $database = $this->databasesByUuid->get($databaseUuid);
             if ($database) {
                 if (! str($database->status)->startsWith('exited')) {
                     $database->update([

app/Jobs/ScheduledJobManager.php

@@ -6,14 +6,15 @@ use App\Models\ScheduledDatabaseBackup;
 use App\Models\ScheduledTask;
 use App\Models\Server;
 use App\Models\Team;
+use Cron\CronExpression;
 use Illuminate\Bus\Queueable;
 use Illuminate\Contracts\Queue\ShouldQueue;
+use Illuminate\Database\Eloquent\Builder;
 use Illuminate\Foundation\Bus\Dispatchable;
 use Illuminate\Queue\InteractsWithQueue;
 use Illuminate\Queue\Middleware\WithoutOverlapping;
 use Illuminate\Queue\SerializesModels;
 use Illuminate\Support\Carbon;
-use Illuminate\Support\Collection;
 use Illuminate\Support\Facades\Cache;
 use Illuminate\Support\Facades\Log;
 use Illuminate\Support\Facades\Redis;
@@ -22,6 +23,8 @@ class ScheduledJobManager implements ShouldQueue
 {
     use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
 
+    private const CHUNK_SIZE = 100;
+
     /**
      * The time when this job execution started.
      * Used to ensure all scheduled items are evaluated against the same point in time.
@@ -37,17 +40,7 @@ class ScheduledJobManager implements ShouldQueue
      */
     public function __construct()
     {
-        $this->onQueue($this->determineQueue());
-    }
-
-    private function determineQueue(): string
-    {
-        $preferredQueue = 'crons';
-        $fallbackQueue = 'high';
-
-        $configuredQueues = explode(',', env('HORIZON_QUEUES', 'high,default'));
-
-        return in_array($preferredQueue, $configuredQueues) ? $preferredQueue : $fallbackQueue;
+        $this->onQueue(crons_queue());
     }
 
     /**
@@ -106,21 +99,11 @@ class ScheduledJobManager implements ShouldQueue
             'execution_time' => $this->executionTime->toIso8601String(),
         ]);
 
-        // Process backups - don't let failures stop task processing
+        // Process scheduled backups and tasks together so neither type starves the other.
         try {
-            $this->processScheduledBackups();
+            $this->processScheduledBackupsAndTasks();
         } catch (\Exception $e) {
-            Log::channel('scheduled-errors')->error('Failed to process scheduled backups', [
-                'error' => $e->getMessage(),
-                'trace' => $e->getTraceAsString(),
-            ]);
-        }
-
-        // Process tasks - don't let failures stop the job manager
-        try {
-            $this->processScheduledTasks();
-        } catch (\Exception $e) {
-            Log::channel('scheduled-errors')->error('Failed to process scheduled tasks', [
+            Log::channel('scheduled-errors')->error('Failed to process scheduled backups and tasks', [
                 'error' => $e->getMessage(),
                 'trace' => $e->getTraceAsString(),
             ]);
@@ -151,125 +134,211 @@ class ScheduledJobManager implements ShouldQueue
         }
     }
 
-    private function processScheduledBackups(): void
+    private function processScheduledBackupsAndTasks(): void
     {
-        $backups = ScheduledDatabaseBackup::with(['database'])
+        $lastBackupId = 0;
+        $lastTaskId = 0;
+
+        do {
+            $backups = $this->scheduledBackupQuery($lastBackupId)->get();
+            $tasks = $this->scheduledTaskQuery($lastTaskId)->get();
+
+            if ($backups->isNotEmpty()) {
+                $lastBackupId = $backups->last()->id;
+            }
+
+            if ($tasks->isNotEmpty()) {
+                $lastTaskId = $tasks->last()->id;
+            }
+
+            $this->processInterleavedDueSchedules(
+                $this->dueScheduledBackups($backups),
+                $this->dueScheduledTasks($tasks),
+            );
+        } while ($backups->isNotEmpty() || $tasks->isNotEmpty());
+    }
+
+    /**
+     * @param  array<int, array{backup: ScheduledDatabaseBackup, server: Server}>  $dueBackups
+     * @param  array<int, array{task: ScheduledTask, server: Server}>  $dueTasks
+     */
+    private function processInterleavedDueSchedules(array $dueBackups, array $dueTasks): void
+    {
+        $maxCount = max(count($dueBackups), count($dueTasks));
+
+        for ($index = 0; $index < $maxCount; $index++) {
+            if (isset($dueBackups[$index])) {
+                $this->processScheduledBackup($dueBackups[$index]['backup'], $dueBackups[$index]['server']);
+            }
+
+            if (isset($dueTasks[$index])) {
+                $this->processScheduledTask($dueTasks[$index]['task'], $dueTasks[$index]['server']);
+            }
+        }
+    }
+
+    private function scheduledBackupQuery(int $lastBackupId): Builder
+    {
+        return ScheduledDatabaseBackup::with(['database', 'team.subscription'])
             ->where('enabled', true)
-            ->get();
+            ->where('id', '>', $lastBackupId)
+            ->orderBy('id')
+            ->limit(self::CHUNK_SIZE);
+    }
+
+    private function scheduledTaskQuery(int $lastTaskId): Builder
+    {
+        return ScheduledTask::with([
+            'service.destination.server.settings',
+            'service.destination.server.team.subscription',
+            'application.destination.server.settings',
+            'application.destination.server.team.subscription',
+        ])
+            ->where('enabled', true)
+            ->where('id', '>', $lastTaskId)
+            ->orderBy('id')
+            ->limit(self::CHUNK_SIZE);
+    }
+
+    /**
+     * @param  iterable<ScheduledDatabaseBackup>  $backups
+     * @return array<int, array{backup: ScheduledDatabaseBackup, server: Server}>
+     */
+    private function dueScheduledBackups(iterable $backups): array
+    {
+        $dueBackups = [];
 
         foreach ($backups as $backup) {
             try {
                 $server = $backup->server();
-                $skipReason = $this->getBackupSkipReason($backup, $server);
-                if ($skipReason !== null) {
-                    $this->skippedCount++;
-                    $this->logSkip('backup', $skipReason, [
-                        'backup_id' => $backup->id,
-                        'database_id' => $backup->database_id,
-                        'database_type' => $backup->database_type,
-                        'team_id' => $backup->team_id ?? null,
-                    ]);
+
+                if (blank(data_get($backup, 'database')) || blank($server)) {
+                    $this->processScheduledBackup($backup, $server);
 
                     continue;
                 }
 
-                $serverTimezone = data_get($server->settings, 'server_timezone', config('app.timezone'));
-
-                if (validate_timezone($serverTimezone) === false) {
-                    $serverTimezone = config('app.timezone');
-                }
-
-                $frequency = $backup->frequency;
-                if (isset(VALID_CRON_STRINGS[$frequency])) {
-                    $frequency = VALID_CRON_STRINGS[$frequency];
-                }
-
-                if (shouldRunCronNow($frequency, $serverTimezone, "scheduled-backup:{$backup->id}", $this->executionTime)) {
-                    DatabaseBackupJob::dispatch($backup);
-                    $this->dispatchedCount++;
-                    Log::channel('scheduled')->info('Backup dispatched', [
-                        'backup_id' => $backup->id,
-                        'database_id' => $backup->database_id,
-                        'database_type' => $backup->database_type,
-                        'team_id' => $backup->team_id ?? null,
-                        'server_id' => $server->id,
-                    ]);
+                if ($this->isDueCandidateBeforeExpensiveChecks($backup->frequency, $server, "scheduled-backup:{$backup->id}")) {
+                    $dueBackups[] = [
+                        'backup' => $backup,
+                        'server' => $server,
+                    ];
                 }
             } catch (\Exception $e) {
-                Log::channel('scheduled-errors')->error('Error processing backup', [
+                Log::channel('scheduled-errors')->error('Error prechecking backup', [
                     'backup_id' => $backup->id,
                     'error' => $e->getMessage(),
                 ]);
             }
         }
+
+        return $dueBackups;
     }
 
-    private function processScheduledTasks(): void
+    /**
+     * @param  iterable<ScheduledTask>  $tasks
+     * @return array<int, array{task: ScheduledTask, server: Server}>
+     */
+    private function dueScheduledTasks(iterable $tasks): array
     {
-        $tasks = ScheduledTask::with(['service', 'application'])
-            ->where('enabled', true)
-            ->get();
+        $dueTasks = [];
 
         foreach ($tasks as $task) {
             try {
                 $server = $task->server();
 
-                // Phase 1: Critical checks (always — cheap, handles orphans and infra issues)
-                $criticalSkip = $this->getTaskCriticalSkipReason($task, $server);
-                if ($criticalSkip !== null) {
-                    $this->skippedCount++;
-                    $this->logSkip('task', $criticalSkip, [
-                        'task_id' => $task->id,
-                        'task_name' => $task->name,
-                        'team_id' => $server?->team_id,
-                    ]);
+                if (blank($server) || (! $task->service && ! $task->application)) {
+                    $this->processScheduledTask($task, $server);
 
                     continue;
                 }
 
-                $serverTimezone = data_get($server->settings, 'server_timezone', config('app.timezone'));
-
-                if (validate_timezone($serverTimezone) === false) {
-                    $serverTimezone = config('app.timezone');
+                if ($this->isDueCandidateBeforeExpensiveChecks($task->frequency, $server, "scheduled-task:{$task->id}")) {
+                    $dueTasks[] = [
+                        'task' => $task,
+                        'server' => $server,
+                    ];
                 }
-
-                $frequency = $task->frequency;
-                if (isset(VALID_CRON_STRINGS[$frequency])) {
-                    $frequency = VALID_CRON_STRINGS[$frequency];
-                }
-
-                if (! shouldRunCronNow($frequency, $serverTimezone, "scheduled-task:{$task->id}", $this->executionTime)) {
-                    continue;
-                }
-
-                // Phase 2: Runtime checks (only when cron is due — avoids noise for stopped resources)
-                $runtimeSkip = $this->getTaskRuntimeSkipReason($task);
-                if ($runtimeSkip !== null) {
-                    $this->skippedCount++;
-                    $this->logSkip('task', $runtimeSkip, [
-                        'task_id' => $task->id,
-                        'task_name' => $task->name,
-                        'team_id' => $server->team_id,
-                    ]);
-
-                    continue;
-                }
-
-                ScheduledTaskJob::dispatch($task);
-                $this->dispatchedCount++;
-                Log::channel('scheduled')->info('Task dispatched', [
-                    'task_id' => $task->id,
-                    'task_name' => $task->name,
-                    'team_id' => $server->team_id,
-                    'server_id' => $server->id,
-                ]);
             } catch (\Exception $e) {
-                Log::channel('scheduled-errors')->error('Error processing task', [
+                Log::channel('scheduled-errors')->error('Error prechecking task', [
                     'task_id' => $task->id,
                     'error' => $e->getMessage(),
                 ]);
             }
         }
+
+        return $dueTasks;
+    }
+
+    private function processScheduledBackup(ScheduledDatabaseBackup $backup, ?Server $precheckedServer = null): void
+    {
+        try {
+            $server = $precheckedServer ?? $backup->server();
+            $skipReason = $this->getBackupSkipReason($backup, $server);
+            if ($skipReason !== null) {
+                $this->skippedCount++;
+                $this->logBackupSkip($backup, $skipReason);
+
+                return;
+            }
+
+            if ($this->shouldDispatch($backup->frequency, $server, "scheduled-backup:{$backup->id}")) {
+                DatabaseBackupJob::dispatch($backup);
+                $this->dispatchedCount++;
+                Log::channel('scheduled')->info('Backup dispatched', [
+                    'backup_id' => $backup->id,
+                    'database_id' => $backup->database_id,
+                    'database_type' => $backup->database_type,
+                    'team_id' => $backup->team_id ?? null,
+                    'server_id' => $server->id,
+                ]);
+            }
+        } catch (\Exception $e) {
+            Log::channel('scheduled-errors')->error('Error processing backup', [
+                'backup_id' => $backup->id,
+                'error' => $e->getMessage(),
+            ]);
+        }
+    }
+
+    private function processScheduledTask(ScheduledTask $task, ?Server $precheckedServer = null): void
+    {
+        try {
+            $server = $precheckedServer ?? $task->server();
+            $criticalSkip = $this->getTaskCriticalSkipReason($task, $server);
+            if ($criticalSkip !== null) {
+                $this->skippedCount++;
+                $this->logTaskSkip($task, $criticalSkip, $server);
+
+                return;
+            }
+
+            if (! $this->shouldDispatch($task->frequency, $server, "scheduled-task:{$task->id}")) {
+                return;
+            }
+
+            $runtimeSkip = $this->getTaskRuntimeSkipReason($task);
+            if ($runtimeSkip !== null) {
+                $this->skippedCount++;
+                $this->logTaskSkip($task, $runtimeSkip, $server);
+
+                return;
+            }
+
+            ScheduledTaskJob::dispatch($task);
+            $this->dispatchedCount++;
+            Log::channel('scheduled')->info('Task dispatched', [
+                'task_id' => $task->id,
+                'task_name' => $task->name,
+                'team_id' => $server->team_id,
+                'server_id' => $server->id,
+            ]);
+        } catch (\Exception $e) {
+            Log::channel('scheduled-errors')->error('Error processing task', [
+                'task_id' => $task->id,
+                'error' => $e->getMessage(),
+            ]);
+        }
     }
 
     private function getBackupSkipReason(ScheduledDatabaseBackup $backup, ?Server $server): ?string
@@ -337,71 +406,70 @@ class ScheduledJobManager implements ShouldQueue
 
     private function processDockerCleanups(): void
     {
-        // Get all servers that need cleanup checks
-        $servers = $this->getServersForCleanup();
-
-        foreach ($servers as $server) {
-            try {
-                $skipReason = $this->getDockerCleanupSkipReason($server);
-                if ($skipReason !== null) {
-                    $this->skippedCount++;
-                    $this->logSkip('docker_cleanup', $skipReason, [
-                        'server_id' => $server->id,
-                        'server_name' => $server->name,
-                        'team_id' => $server->team_id,
-                    ]);
-
-                    continue;
+        $this->getServersForCleanupQuery()
+            ->chunkById(self::CHUNK_SIZE, function ($servers): void {
+                foreach ($servers as $server) {
+                    $this->processDockerCleanup($server);
                 }
+            });
+    }
 
-                $serverTimezone = data_get($server->settings, 'server_timezone', config('app.timezone'));
-                if (validate_timezone($serverTimezone) === false) {
-                    $serverTimezone = config('app.timezone');
-                }
-
-                $frequency = data_get($server->settings, 'docker_cleanup_frequency', '0 * * * *');
-                if (isset(VALID_CRON_STRINGS[$frequency])) {
-                    $frequency = VALID_CRON_STRINGS[$frequency];
-                }
-
-                // Use the frozen execution time for consistent evaluation
-                if (shouldRunCronNow($frequency, $serverTimezone, "docker-cleanup:{$server->id}", $this->executionTime)) {
-                    DockerCleanupJob::dispatch(
-                        $server,
-                        false,
-                        $server->settings->delete_unused_volumes,
-                        $server->settings->delete_unused_networks
-                    );
-                    $this->dispatchedCount++;
-                    Log::channel('scheduled')->info('Docker cleanup dispatched', [
-                        'server_id' => $server->id,
-                        'server_name' => $server->name,
-                        'team_id' => $server->team_id,
-                    ]);
-                }
-            } catch (\Exception $e) {
-                Log::channel('scheduled-errors')->error('Error processing docker cleanup', [
+    private function processDockerCleanup(Server $server): void
+    {
+        try {
+            $skipReason = $this->getDockerCleanupSkipReason($server);
+            if ($skipReason !== null) {
+                $this->skippedCount++;
+                $this->logSkip('docker_cleanup', $skipReason, [
                     'server_id' => $server->id,
                     'server_name' => $server->name,
-                    'error' => $e->getMessage(),
+                    'team_id' => $server->team_id,
+                ]);
+
+                return;
+            }
+
+            $frequency = data_get($server->settings, 'docker_cleanup_frequency', '0 * * * *');
+
+            if ($this->shouldDispatch($frequency, $server, "docker-cleanup:{$server->id}")) {
+                DockerCleanupJob::dispatch(
+                    $server,
+                    false,
+                    $server->settings->delete_unused_volumes,
+                    $server->settings->delete_unused_networks
+                );
+                $this->dispatchedCount++;
+                Log::channel('scheduled')->info('Docker cleanup dispatched', [
+                    'server_id' => $server->id,
+                    'server_name' => $server->name,
+                    'team_id' => $server->team_id,
                 ]);
             }
+        } catch (\Exception $e) {
+            Log::channel('scheduled-errors')->error('Error processing docker cleanup', [
+                'server_id' => $server->id,
+                'server_name' => $server->name,
+                'error' => $e->getMessage(),
+            ]);
         }
     }
 
-    private function getServersForCleanup(): Collection
+    private function getServersForCleanupQuery(): Builder
     {
         $query = Server::with('settings')
             ->where('ip', '!=', '1.2.3.4');
 
         if (isCloud()) {
-            $servers = $query->whereRelation('team.subscription', 'stripe_invoice_paid', true)->get();
-            $own = Team::find(0)->servers()->with('settings')->get();
-
-            return $servers->merge($own);
+            $query
+                ->with('team.subscription')
+                ->where(function (Builder $query): void {
+                    $query
+                        ->where('team_id', 0)
+                        ->orWhereRelation('team.subscription', 'stripe_invoice_paid', true);
+                });
         }
 
-        return $query->get();
+        return $query;
     }
 
     private function getDockerCleanupSkipReason(Server $server): ?string
@@ -428,4 +496,71 @@ class ScheduledJobManager implements ShouldQueue
             'execution_time' => $this->executionTime?->toIso8601String(),
         ], $context));
     }
+
+    private function shouldDispatch(string $frequency, Server $server, string $dedupKey): bool
+    {
+        return shouldRunCronNow(
+            $this->normalizeFrequency($frequency),
+            $this->serverTimezone($server),
+            $dedupKey,
+            $this->executionTime,
+        );
+    }
+
+    private function isDueCandidateBeforeExpensiveChecks(string $frequency, Server $server, string $dedupKey): bool
+    {
+        $cron = new CronExpression($this->normalizeFrequency($frequency));
+        $executionTime = ($this->executionTime ?? Carbon::now())->copy()->setTimezone($this->serverTimezone($server));
+        $lastDispatched = Cache::get($dedupKey);
+        $previousDue = Carbon::instance($cron->getPreviousRunDate($executionTime, allowCurrentDate: true));
+
+        if ($lastDispatched === null) {
+            $isDue = $cron->isDue($executionTime);
+
+            if (! $isDue) {
+                Cache::put($dedupKey, $previousDue->toIso8601String(), 2592000);
+            }
+
+            return $isDue;
+        }
+
+        $shouldFire = $previousDue->gt(Carbon::parse($lastDispatched));
+
+        if (! $shouldFire) {
+            Cache::put($dedupKey, $previousDue->toIso8601String(), 2592000);
+        }
+
+        return $shouldFire;
+    }
+
+    private function normalizeFrequency(string $frequency): string
+    {
+        return VALID_CRON_STRINGS[$frequency] ?? $frequency;
+    }
+
+    private function serverTimezone(Server $server): string
+    {
+        $timezone = data_get($server->settings, 'server_timezone', config('app.timezone'));
+
+        return validate_timezone($timezone) ? $timezone : config('app.timezone');
+    }
+
+    private function logBackupSkip(ScheduledDatabaseBackup $backup, string $reason): void
+    {
+        $this->logSkip('backup', $reason, [
+            'backup_id' => $backup->id,
+            'database_id' => $backup->database_id,
+            'database_type' => $backup->database_type,
+            'team_id' => $backup->team_id ?? null,
+        ]);
+    }
+
+    private function logTaskSkip(ScheduledTask $task, string $reason, ?Server $server): void
+    {
+        $this->logSkip('task', $reason, [
+            'task_id' => $task->id,
+            'task_name' => $task->name,
+            'team_id' => $server?->team_id,
+        ]);
+    }
 }

app/Jobs/ScheduledTaskJob.php

@@ -40,13 +40,13 @@ class ScheduledTaskJob implements ShouldBeEncrypted, ShouldQueue
      */
     public $timeout = 300;
 
-    public Team $team;
+    public ?Team $team = null;
 
     public ?Server $server = null;
 
     public ScheduledTask $task;
 
-    public Application|Service $resource;
+    public Application|Service|null $resource = null;
 
     public ?ScheduledTaskExecution $task_log = null;
 
@@ -61,25 +61,34 @@ class ScheduledTaskJob implements ShouldBeEncrypted, ShouldQueue
 
     public array $containers = [];
 
-    public string $server_timezone;
+    public string $server_timezone = 'UTC';
 
-    public function __construct($task)
+    public function __construct(ScheduledTask $task)
     {
-        $this->onQueue('high');
+        $this->onQueue(crons_queue());
 
         $this->task = $task;
-        if ($service = $task->service()->first()) {
-            $this->resource = $service;
-        } elseif ($application = $task->application()->first()) {
-            $this->resource = $application;
+        $this->timeout = $this->task->timeout ?? 300;
+    }
+
+    private function initializeExecutionContext(): void
+    {
+        $this->task->loadMissing([
+            'service.destination.server.settings',
+            'application.destination.server.settings',
+        ]);
+
+        if ($this->task->service) {
+            $this->resource = $this->task->service;
+        } elseif ($this->task->application) {
+            $this->resource = $this->task->application;
         } else {
             throw new \RuntimeException('ScheduledTaskJob failed: No resource found.');
         }
-        $this->team = Team::findOrFail($task->team_id);
-        $this->server_timezone = $this->getServerTimezone();
 
-        // Set timeout from task configuration
-        $this->timeout = $this->task->timeout ?? 300;
+        $this->team = Team::findOrFail($this->task->team_id);
+        $this->server_timezone = $this->getServerTimezone();
+        $this->server = $this->resource->destination->server;
     }
 
     private function getServerTimezone(): string
@@ -98,6 +107,8 @@ class ScheduledTaskJob implements ShouldBeEncrypted, ShouldQueue
         $startTime = Carbon::now();
 
         try {
+            $this->initializeExecutionContext();
+
             $this->task_log = ScheduledTaskExecution::create([
                 'scheduled_task_id' => $this->task->id,
                 'started_at' => $startTime,
@@ -107,8 +118,6 @@ class ScheduledTaskJob implements ShouldBeEncrypted, ShouldQueue
             // Store execution ID for timeout handling
             $this->executionId = $this->task_log->id;
 
-            $this->server = $this->resource->destination->server;
-
             if ($this->resource->type() === 'application') {
                 $containers = getCurrentApplicationContainerStatus($this->server, $this->resource->id, 0);
                 if ($containers->count() > 0) {
@@ -179,7 +188,10 @@ class ScheduledTaskJob implements ShouldBeEncrypted, ShouldQueue
             // Re-throw to trigger Laravel's retry mechanism with backoff
             throw $e;
         } finally {
-            ScheduledTaskDone::dispatch($this->team->id);
+            if ($this->team) {
+                ScheduledTaskDone::dispatch($this->team->id);
+            }
+
             if ($this->task_log) {
                 $finishedAt = Carbon::now();
                 $duration = round($startTime->floatDiffInSeconds($finishedAt), 2);
@@ -205,6 +217,8 @@ class ScheduledTaskJob implements ShouldBeEncrypted, ShouldQueue
      */
     public function failed(?\Throwable $exception): void
     {
+        $this->team ??= Team::find($this->task->team_id);
+
         Log::channel('scheduled-errors')->error('ScheduledTask permanently failed', [
             'job' => 'ScheduledTaskJob',
             'task_id' => $this->task->uuid,

app/Jobs/SendWebhookJob.php

@@ -2,6 +2,7 @@
 
 namespace App\Jobs;
 
+use App\Rules\SafeWebhookUrl;
 use Illuminate\Bus\Queueable;
 use Illuminate\Contracts\Queue\ShouldBeEncrypted;
 use Illuminate\Contracts\Queue\ShouldQueue;
@@ -9,6 +10,8 @@ use Illuminate\Foundation\Bus\Dispatchable;
 use Illuminate\Queue\InteractsWithQueue;
 use Illuminate\Queue\SerializesModels;
 use Illuminate\Support\Facades\Http;
+use Illuminate\Support\Facades\Log;
+use Illuminate\Support\Facades\Validator;
 
 class SendWebhookJob implements ShouldBeEncrypted, ShouldQueue
 {
@@ -40,6 +43,20 @@ class SendWebhookJob implements ShouldBeEncrypted, ShouldQueue
      */
     public function handle(): void
     {
+        $validator = Validator::make(
+            ['webhook_url' => $this->webhookUrl],
+            ['webhook_url' => ['required', 'url', new SafeWebhookUrl]]
+        );
+
+        if ($validator->fails()) {
+            Log::warning('SendWebhookJob: blocked unsafe webhook URL', [
+                'url' => $this->webhookUrl,
+                'errors' => $validator->errors()->all(),
+            ]);
+
+            return;
+        }
+
         if (isDev()) {
             ray('Sending webhook notification', [
                 'url' => $this->webhookUrl,

app/Jobs/ServerCheckJob.php

@@ -15,6 +15,7 @@ use Illuminate\Foundation\Bus\Dispatchable;
 use Illuminate\Queue\InteractsWithQueue;
 use Illuminate\Queue\Middleware\WithoutOverlapping;
 use Illuminate\Queue\SerializesModels;
+use Illuminate\Queue\TimeoutExceededException;
 use Illuminate\Support\Facades\Log;
 
 class ServerCheckJob implements ShouldBeEncrypted, ShouldQueue
@@ -36,11 +37,12 @@ class ServerCheckJob implements ShouldBeEncrypted, ShouldQueue
 
     public function failed(?\Throwable $exception): void
     {
-        if ($exception instanceof \Illuminate\Queue\TimeoutExceededException) {
+        if ($exception instanceof TimeoutExceededException) {
             Log::warning('ServerCheckJob timed out', [
                 'server_id' => $this->server->id,
                 'server_name' => $this->server->name,
             ]);
+            $this->server->increment('unreachable_count');
 
             // Delete the queue job so it doesn't appear in Horizon's failed list.
             $this->job?->delete();

app/Jobs/ServerConnectionCheckJob.php

@@ -2,8 +2,11 @@
 
 namespace App\Jobs;
 
+use App\Events\ServerReachabilityChanged;
+use App\Helpers\SshMultiplexingHelper;
 use App\Models\Server;
 use App\Services\ConfigurationRepository;
+use App\Services\HetznerService;
 use Illuminate\Bus\Queueable;
 use Illuminate\Contracts\Queue\ShouldBeEncrypted;
 use Illuminate\Contracts\Queue\ShouldQueue;
@@ -11,7 +14,9 @@ use Illuminate\Foundation\Bus\Dispatchable;
 use Illuminate\Queue\InteractsWithQueue;
 use Illuminate\Queue\Middleware\WithoutOverlapping;
 use Illuminate\Queue\SerializesModels;
+use Illuminate\Queue\TimeoutExceededException;
 use Illuminate\Support\Facades\Log;
+use Illuminate\Support\Facades\Process;
 
 class ServerConnectionCheckJob implements ShouldBeEncrypted, ShouldQueue
 {
@@ -19,7 +24,7 @@ class ServerConnectionCheckJob implements ShouldBeEncrypted, ShouldQueue
 
     public $tries = 1;
 
-    public $timeout = 30;
+    public $timeout = 15;
 
     public function __construct(
         public Server $server,
@@ -28,7 +33,7 @@ class ServerConnectionCheckJob implements ShouldBeEncrypted, ShouldQueue
 
     public function middleware(): array
     {
-        return [(new WithoutOverlapping('server-connection-check-'.$this->server->uuid))->expireAfter(45)->dontRelease()];
+        return [(new WithoutOverlapping('server-connection-check-'.$this->server->uuid))->expireAfter(25)->dontRelease()];
     }
 
     private function disableSshMux(): void
@@ -39,6 +44,9 @@ class ServerConnectionCheckJob implements ShouldBeEncrypted, ShouldQueue
 
     public function handle()
     {
+        $wasReachable = (bool) $this->server->settings->is_reachable;
+        $wasNotified = (bool) $this->server->unreachable_notification_sent;
+
         try {
             // Check if server is disabled
             if ($this->server->settings->force_disabled) {
@@ -72,6 +80,7 @@ class ServerConnectionCheckJob implements ShouldBeEncrypted, ShouldQueue
                     'is_reachable' => false,
                     'is_usable' => false,
                 ]);
+                $this->server->increment('unreachable_count');
 
                 Log::warning('ServerConnectionCheck: Server not reachable', [
                     'server_id' => $this->server->id,
@@ -79,6 +88,8 @@ class ServerConnectionCheckJob implements ShouldBeEncrypted, ShouldQueue
                     'server_ip' => $this->server->ip,
                 ]);
 
+                $this->dispatchReachabilityChangedIfNeeded($wasReachable, $wasNotified, false);
+
                 return;
             }
 
@@ -90,6 +101,12 @@ class ServerConnectionCheckJob implements ShouldBeEncrypted, ShouldQueue
                 'is_usable' => $isUsable,
             ]);
 
+            if ($this->server->unreachable_count > 0) {
+                $this->server->update(['unreachable_count' => 0]);
+            }
+
+            $this->dispatchReachabilityChangedIfNeeded($wasReachable, $wasNotified, true);
+
         } catch (\Throwable $e) {
 
             Log::error('ServerConnectionCheckJob failed', [
@@ -100,6 +117,9 @@ class ServerConnectionCheckJob implements ShouldBeEncrypted, ShouldQueue
                 'is_reachable' => false,
                 'is_usable' => false,
             ]);
+            $this->server->increment('unreachable_count');
+
+            $this->dispatchReachabilityChangedIfNeeded($wasReachable, $wasNotified, false);
 
             return;
         }
@@ -107,23 +127,48 @@ class ServerConnectionCheckJob implements ShouldBeEncrypted, ShouldQueue
 
     public function failed(?\Throwable $exception): void
     {
-        if ($exception instanceof \Illuminate\Queue\TimeoutExceededException) {
+        if ($exception instanceof TimeoutExceededException) {
+            $wasReachable = (bool) $this->server->settings->is_reachable;
+            $wasNotified = (bool) $this->server->unreachable_notification_sent;
+
             $this->server->settings->update([
                 'is_reachable' => false,
                 'is_usable' => false,
             ]);
+            $this->server->increment('unreachable_count');
+
+            $this->dispatchReachabilityChangedIfNeeded($wasReachable, $wasNotified, false);
 
             // Delete the queue job so it doesn't appear in Horizon's failed list.
             $this->job?->delete();
         }
     }
 
+    /**
+     * Fire ServerReachabilityChanged when state crosses the unreachable threshold (count >= 2)
+     * or when a previously-notified server recovers. Skips noise from single transient flaps.
+     */
+    private function dispatchReachabilityChangedIfNeeded(bool $wasReachable, bool $wasNotified, bool $isReachable): void
+    {
+        if ($isReachable) {
+            if (! $wasReachable || $wasNotified) {
+                ServerReachabilityChanged::dispatch($this->server);
+            }
+
+            return;
+        }
+
+        if ($this->server->unreachable_count >= 2 && ! $wasNotified) {
+            ServerReachabilityChanged::dispatch($this->server);
+        }
+    }
+
     private function checkHetznerStatus(): void
     {
         $status = null;
 
         try {
-            $hetznerService = new \App\Services\HetznerService($this->server->cloudProviderToken->token);
+            $hetznerService = new HetznerService($this->server->cloudProviderToken->token);
             $serverData = $hetznerService->getServer($this->server->hetzner_server_id);
             $status = $serverData['status'] ?? null;
 
@@ -144,15 +189,18 @@ class ServerConnectionCheckJob implements ShouldBeEncrypted, ShouldQueue
     private function checkConnection(): bool
     {
         try {
-            // Use instant_remote_process with a simple command
-            // This will automatically handle mux, sudo, IPv6, Cloudflare tunnel, etc.
-            $output = instant_remote_process_with_timeout(
-                ['ls -la /'],
-                $this->server,
-                false // don't throw error
-            );
+            // Single SSH attempt without SshRetryHandler — retries waste time for connectivity checks.
+            // Backoff is managed at the dispatch level via unreachable_count.
+            $commands = ['ls -la /'];
+            if ($this->server->isNonRoot()) {
+                $commands = parseCommandsByLineForSudo(collect($commands), $this->server);
+            }
+            $commandString = implode("\n", $commands);
 
-            return $output !== null;
+            $sshCommand = SshMultiplexingHelper::generateSshCommand($this->server, $commandString, true);
+            $process = Process::timeout(10)->run($sshCommand);
+
+            return $process->exitCode() === 0;
         } catch (\Throwable $e) {
             Log::debug('ServerConnectionCheck: Connection check failed', [
                 'server_id' => $this->server->id,

app/Jobs/ServerManagerJob.php

@@ -86,6 +86,9 @@ class ServerManagerJob implements ShouldBeEncrypted, ShouldQueue
                     if ($server->isSentinelEnabled() && $server->isSentinelLive()) {
                         return;
                     }
+                    if ($this->shouldSkipDueToBackoff($server)) {
+                        return;
+                    }
                     ServerConnectionCheckJob::dispatch($server);
                 } catch (\Exception $e) {
                     Log::channel('scheduled-errors')->error('Failed to dispatch ServerConnectionCheck', [
@@ -129,7 +132,9 @@ class ServerManagerJob implements ShouldBeEncrypted, ShouldQueue
         if ($sentinelOutOfSync) {
             // Dispatch ServerCheckJob if Sentinel is out of sync
             if (shouldRunCronNow($this->checkFrequency, $serverTimezone, "server-check:{$server->id}", $this->executionTime)) {
-                ServerCheckJob::dispatch($server);
+                if (! $this->shouldSkipDueToBackoff($server)) {
+                    ServerCheckJob::dispatch($server);
+                }
             }
         }
 
@@ -165,4 +170,39 @@ class ServerManagerJob implements ShouldBeEncrypted, ShouldQueue
         // Note: CheckAndStartSentinelJob is only dispatched daily (line above) for version updates.
         // Crash recovery is handled by sentinelOutOfSync → ServerCheckJob → CheckAndStartSentinelJob.
     }
+
+    /**
+     * Determine the backoff cycle interval based on how many consecutive times a server has been unreachable.
+     * Higher counts → less frequent checks (based on 5-min cloud cycle):
+     *   0-2: every cycle, 3-5: ~15 min, 6-11: ~30 min, 12+: ~60 min
+     */
+    private function getBackoffCycleInterval(int $unreachableCount): int
+    {
+        return match (true) {
+            $unreachableCount <= 2 => 1,
+            $unreachableCount <= 5 => 3,
+            $unreachableCount <= 11 => 6,
+            default => 12,
+        };
+    }
+
+    /**
+     * Check if a server should be skipped this cycle due to unreachable backoff.
+     * Uses server ID hash to distribute checks across cycles (avoid thundering herd).
+     */
+    private function shouldSkipDueToBackoff(Server $server): bool
+    {
+        $unreachableCount = $server->unreachable_count ?? 0;
+        $interval = $this->getBackoffCycleInterval($unreachableCount);
+
+        if ($interval <= 1) {
+            return false;
+        }
+
+        $cyclePeriodMinutes = isCloud() ? 5 : 1;
+        $cycleIndex = intdiv($this->executionTime->minute, $cyclePeriodMinutes);
+        $serverHash = abs(crc32((string) $server->id));
+
+        return ($cycleIndex + $serverHash) % $interval !== 0;
+    }
 }

app/Jobs/StripeProcessJob.php

@@ -9,6 +9,7 @@ use Illuminate\Contracts\Queue\ShouldBeEncrypted;
 use Illuminate\Contracts\Queue\ShouldQueue;
 use Illuminate\Foundation\Queue\Queueable;
 use Illuminate\Support\Str;
+use Stripe\StripeClient;
 
 class StripeProcessJob implements ShouldBeEncrypted, ShouldQueue
 {
@@ -35,7 +36,7 @@ class StripeProcessJob implements ShouldBeEncrypted, ShouldQueue
             $data = data_get($this->event, 'data.object');
             switch ($type) {
                 case 'radar.early_fraud_warning.created':
-                    $stripe = new \Stripe\StripeClient(config('subscription.stripe_api_key'));
+                    $stripe = new StripeClient(config('subscription.stripe_api_key'));
                     $id = data_get($data, 'id');
                     $charge = data_get($data, 'charge');
                     if ($charge) {
@@ -94,12 +95,12 @@ class StripeProcessJob implements ShouldBeEncrypted, ShouldQueue
                     }
                     $subscription = Subscription::where('stripe_customer_id', $customerId)->first();
                     if (! $subscription) {
-                        throw new \RuntimeException("No subscription found for customer: {$customerId}");
+                        break;
                     }
 
                     if ($subscription->stripe_subscription_id) {
                         try {
-                            $stripe = new \Stripe\StripeClient(config('subscription.stripe_api_key'));
+                            $stripe = new StripeClient(config('subscription.stripe_api_key'));
                             $stripeSubscription = $stripe->subscriptions->retrieve(
                                 $subscription->stripe_subscription_id
                             );
@@ -154,7 +155,7 @@ class StripeProcessJob implements ShouldBeEncrypted, ShouldQueue
                     $subscription = Subscription::where('stripe_customer_id', $customerId)->first();
                     if (! $subscription) {
                         // send_internal_notification('invoice.payment_failed failed but no subscription found in Coolify for customer: '.$customerId);
-                        throw new \RuntimeException("No subscription found for customer: {$customerId}");
+                        break;
                     }
                     $team = data_get($subscription, 'team');
                     if (! $team) {
@@ -165,7 +166,7 @@ class StripeProcessJob implements ShouldBeEncrypted, ShouldQueue
                     // Verify payment status with Stripe API before sending failure notification
                     if ($paymentIntentId) {
                         try {
-                            $stripe = new \Stripe\StripeClient(config('subscription.stripe_api_key'));
+                            $stripe = new StripeClient(config('subscription.stripe_api_key'));
                             $paymentIntent = $stripe->paymentIntents->retrieve($paymentIntentId);
 
                             if (in_array($paymentIntent->status, ['processing', 'succeeded', 'requires_action', 'requires_confirmation'])) {
@@ -190,7 +191,7 @@ class StripeProcessJob implements ShouldBeEncrypted, ShouldQueue
                     $subscription = Subscription::where('stripe_customer_id', $customerId)->first();
                     if (! $subscription) {
                         // send_internal_notification('payment_intent.payment_failed, no subscription found in Coolify for customer: '.$customerId);
-                        throw new \RuntimeException("No subscription found in Coolify for customer: {$customerId}");
+                        break;
                     }
                     if ($subscription->stripe_invoice_paid) {
                         // send_internal_notification('payment_intent.payment_failed but invoice is active for customer: '.$customerId);
@@ -334,7 +335,7 @@ class StripeProcessJob implements ShouldBeEncrypted, ShouldQueue
                         }
                     } else {
                         // send_internal_notification('Subscription deleted but no subscription found in Coolify for customer: '.$customerId);
-                        throw new \RuntimeException("No subscription found in Coolify for customer: {$customerId}");
+                        break;
                     }
                     break;
                 default:

app/Jobs/ValidateAndInstallServerJob.php

@@ -45,7 +45,8 @@ class ValidateAndInstallServerJob implements ShouldBeEncrypted, ShouldQueue
             // Validate connection
             ['uptime' => $uptime, 'error' => $error] = $this->server->validateConnection();
             if (! $uptime) {
-                $errorMessage = 'Server is not reachable. Please validate your configuration and connection.<br>Check this <a target="_blank" class="underline" href="https://coolify.io/docs/knowledge-base/server/openssh">documentation</a> for further help. <br><br>Error: '.$error;
+                $sanitizedError = htmlspecialchars($error ?? '', ENT_QUOTES, 'UTF-8');
+                $errorMessage = 'Server is not reachable. Please validate your configuration and connection.<br>Check this <a target="_blank" class="underline" href="https://coolify.io/docs/knowledge-base/server/openssh">documentation</a> for further help. <br><br>Error: '.$sanitizedError;
                 $this->server->update([
                     'validation_logs' => $errorMessage,
                     'is_validating' => false,
@@ -197,7 +198,7 @@ class ValidateAndInstallServerJob implements ShouldBeEncrypted, ShouldQueue
             ]);
 
             $this->server->update([
-                'validation_logs' => 'An error occurred during validation: '.$e->getMessage(),
+                'validation_logs' => 'An error occurred during validation: '.htmlspecialchars($e->getMessage(), ENT_QUOTES, 'UTF-8'),
                 'is_validating' => false,
             ]);
         }

app/Jobs/VolumeCloneJob.php

@@ -43,27 +43,34 @@ class VolumeCloneJob implements ShouldBeEncrypted, ShouldQueue
 
     protected function cloneLocalVolume()
     {
+        $srcVol = escapeshellarg($this->sourceVolume);
+        $tgtVol = escapeshellarg($this->targetVolume);
+
         instant_remote_process([
-            "docker volume create $this->targetVolume",
-            "docker run --rm -v $this->sourceVolume:/source -v $this->targetVolume:/target alpine sh -c 'cp -a /source/. /target/ && chown -R 1000:1000 /target'",
+            "docker volume create {$tgtVol}",
+            "docker run --rm -v {$srcVol}:/source -v {$tgtVol}:/target alpine sh -c 'cp -a /source/. /target/ && chown -R 1000:1000 /target'",
         ], $this->sourceServer);
     }
 
     protected function cloneRemoteVolume()
     {
+        $srcVol = escapeshellarg($this->sourceVolume);
+        $tgtVol = escapeshellarg($this->targetVolume);
         $sourceCloneDir = "{$this->cloneDir}/{$this->sourceVolume}";
         $targetCloneDir = "{$this->cloneDir}/{$this->targetVolume}";
+        $srcDir = escapeshellarg($sourceCloneDir);
+        $tgtDir = escapeshellarg($targetCloneDir);
 
         try {
             instant_remote_process([
-                "mkdir -p $sourceCloneDir",
-                "chmod 777 $sourceCloneDir",
-                "docker run --rm -v $this->sourceVolume:/source -v $sourceCloneDir:/clone alpine sh -c 'cd /source && tar czf /clone/volume-data.tar.gz .'",
+                "mkdir -p {$srcDir}",
+                "chmod 777 {$srcDir}",
+                "docker run --rm -v {$srcVol}:/source -v {$srcDir}:/clone alpine sh -c 'cd /source && tar czf /clone/volume-data.tar.gz .'",
             ], $this->sourceServer);
 
             instant_remote_process([
-                "mkdir -p $targetCloneDir",
-                "chmod 777 $targetCloneDir",
+                "mkdir -p {$tgtDir}",
+                "chmod 777 {$tgtDir}",
             ], $this->targetServer);
 
             instant_scp(
@@ -74,8 +81,8 @@ class VolumeCloneJob implements ShouldBeEncrypted, ShouldQueue
             );
 
             instant_remote_process([
-                "docker volume create $this->targetVolume",
-                "docker run --rm -v $this->targetVolume:/target -v $targetCloneDir:/clone alpine sh -c 'cd /target && tar xzf /clone/volume-data.tar.gz && chown -R 1000:1000 /target'",
+                "docker volume create {$tgtVol}",
+                "docker run --rm -v {$tgtVol}:/target -v {$tgtDir}:/clone alpine sh -c 'cd /target && tar xzf /clone/volume-data.tar.gz && chown -R 1000:1000 /target'",
             ], $this->targetServer);
 
         } catch (\Exception $e) {
@@ -84,7 +91,7 @@ class VolumeCloneJob implements ShouldBeEncrypted, ShouldQueue
         } finally {
             try {
                 instant_remote_process([
-                    "rm -rf $sourceCloneDir",
+                    "rm -rf {$srcDir}",
                 ], $this->sourceServer, false);
             } catch (\Exception $e) {
                 \Log::warning('Failed to clean up source server clone directory: '.$e->getMessage());
@@ -93,7 +100,7 @@ class VolumeCloneJob implements ShouldBeEncrypted, ShouldQueue
             try {
                 if ($this->targetServer) {
                     instant_remote_process([
-                        "rm -rf $targetCloneDir",
+                        "rm -rf {$tgtDir}",
                     ], $this->targetServer, false);
                 }
             } catch (\Exception $e) {