feat(jobs): add queue delay resilience to scheduled job execution

Implement dedup key-based cron tracking to make scheduled jobs resilient to queue
delays. Even if a job is delayed by minutes, it will catch the missed cron window
by tracking previousRunDate in cache instead of relying on isDue() alone.

- Add dedupKey parameter to shouldRunNow() in ScheduledJobManager
  - When provided, uses getPreviousRunDate() + cache tracking for resilience
  - Falls back to isDue() for docker cleanups without dedup key
  - Prevents double-dispatch within same cron window

- Optimize ServerConnectionCheckJob dispatch
  - Skip SSH checks if Sentinel is healthy (enabled and live)
  - Reduces redundant checks when Sentinel heartbeat proves connectivity

- Remove hourly Sentinel update checks
  - Consolidate to daily CheckAndStartSentinelJob dispatch
  - Crash recovery handled by sentinelOutOfSync → ServerCheckJob flow

- Add logging for skipped database backups with context (backup_id, database_id, status)

- Refactor skip reason methods to accept server parameter, avoiding redundant queries

- Add comprehensive test suite for scheduling with various delay scenarios and timezones

app/Jobs/DatabaseBackupJob.php

@@ -111,6 +111,12 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue
 
             $status = str(data_get($this->database, 'status'));
             if (! $status->startsWith('running') && $this->database->id !== 0) {
+                Log::info('DatabaseBackupJob skipped: database not running', [
+                    'backup_id' => $this->backup->id,
+                    'database_id' => $this->database->id,
+                    'status' => (string) $status,
+                ]);
+
                 return;
             }
             if (data_get($this->backup, 'database_type') === \App\Models\ServiceDatabase::class) {

app/Jobs/ScheduledJobManager.php

@@ -160,7 +160,8 @@ class ScheduledJobManager implements ShouldQueue
 
         foreach ($backups as $backup) {
             try {
-                $skipReason = $this->getBackupSkipReason($backup);
+                $server = $backup->server();
+                $skipReason = $this->getBackupSkipReason($backup, $server);
                 if ($skipReason !== null) {
                     $this->skippedCount++;
                     $this->logSkip('backup', $skipReason, [
@@ -173,7 +174,6 @@ class ScheduledJobManager implements ShouldQueue
                     continue;
                 }
 
-                $server = $backup->server();
                 $serverTimezone = data_get($server->settings, 'server_timezone', config('app.timezone'));
 
                 if (validate_timezone($serverTimezone) === false) {
@@ -185,7 +185,7 @@ class ScheduledJobManager implements ShouldQueue
                     $frequency = VALID_CRON_STRINGS[$frequency];
                 }
 
-                if ($this->shouldRunNow($frequency, $serverTimezone)) {
+                if ($this->shouldRunNow($frequency, $serverTimezone, "scheduled-backup:{$backup->id}")) {
                     DatabaseBackupJob::dispatch($backup);
                     $this->dispatchedCount++;
                     Log::channel('scheduled')->info('Backup dispatched', [
@@ -213,19 +213,19 @@ class ScheduledJobManager implements ShouldQueue
 
         foreach ($tasks as $task) {
             try {
-                $skipReason = $this->getTaskSkipReason($task);
+                $server = $task->server();
+                $skipReason = $this->getTaskSkipReason($task, $server);
                 if ($skipReason !== null) {
                     $this->skippedCount++;
                     $this->logSkip('task', $skipReason, [
                         'task_id' => $task->id,
                         'task_name' => $task->name,
-                        'team_id' => $task->server()?->team_id,
+                        'team_id' => $server?->team_id,
                     ]);
 
                     continue;
                 }
 
-                $server = $task->server();
                 $serverTimezone = data_get($server->settings, 'server_timezone', config('app.timezone'));
 
                 if (validate_timezone($serverTimezone) === false) {
@@ -237,7 +237,7 @@ class ScheduledJobManager implements ShouldQueue
                     $frequency = VALID_CRON_STRINGS[$frequency];
                 }
 
-                if ($this->shouldRunNow($frequency, $serverTimezone)) {
+                if ($this->shouldRunNow($frequency, $serverTimezone, "scheduled-task:{$task->id}")) {
                     ScheduledTaskJob::dispatch($task);
                     $this->dispatchedCount++;
                     Log::channel('scheduled')->info('Task dispatched', [
@@ -256,7 +256,7 @@ class ScheduledJobManager implements ShouldQueue
         }
     }
 
-    private function getBackupSkipReason(ScheduledDatabaseBackup $backup): ?string
+    private function getBackupSkipReason(ScheduledDatabaseBackup $backup, ?Server $server): ?string
     {
         if (blank(data_get($backup, 'database'))) {
             $backup->delete();
@@ -264,7 +264,6 @@ class ScheduledJobManager implements ShouldQueue
             return 'database_deleted';
         }
 
-        $server = $backup->server();
         if (blank($server)) {
             $backup->delete();
 
@@ -282,12 +281,11 @@ class ScheduledJobManager implements ShouldQueue
         return null;
     }
 
-    private function getTaskSkipReason(ScheduledTask $task): ?string
+    private function getTaskSkipReason(ScheduledTask $task, ?Server $server): ?string
     {
         $service = $task->service;
         $application = $task->application;
 
-        $server = $task->server();
         if (blank($server)) {
             $task->delete();
 
@@ -319,16 +317,38 @@ class ScheduledJobManager implements ShouldQueue
         return null;
     }
 
-    private function shouldRunNow(string $frequency, string $timezone): bool
+    /**
+     * Determine if a cron schedule should run now.
+     *
+     * When a dedupKey is provided, uses getPreviousRunDate() + last-dispatch tracking
+     * instead of isDue(). This is resilient to queue delays — even if the job is delayed
+     * by minutes, it still catches the missed cron window. Without dedupKey, falls back
+     * to simple isDue() check.
+     */
+    private function shouldRunNow(string $frequency, string $timezone, ?string $dedupKey = null): bool
     {
         $cron = new CronExpression($frequency);
-
-        // Use the frozen execution time, not the current time
-        // Fallback to current time if execution time is not set (shouldn't happen)
         $baseTime = $this->executionTime ?? Carbon::now();
         $executionTime = $baseTime->copy()->setTimezone($timezone);
 
-        return $cron->isDue($executionTime);
+        // No dedup key → simple isDue check (used by docker cleanups)
+        if ($dedupKey === null) {
+            return $cron->isDue($executionTime);
+        }
+
+        // Get the most recent time this cron was due (including current minute)
+        $previousDue = Carbon::instance($cron->getPreviousRunDate($executionTime, allowCurrentDate: true));
+
+        $lastDispatched = Cache::get($dedupKey);
+
+        // Run if: never dispatched before, OR there's been a due time since last dispatch
+        if ($lastDispatched === null || $previousDue->gt(Carbon::parse($lastDispatched))) {
+            Cache::put($dedupKey, $executionTime->toIso8601String(), 86400);
+
+            return true;
+        }
+
+        return false;
     }
 
     private function processDockerCleanups(): void

app/Jobs/ServerManagerJob.php

@@ -64,11 +64,11 @@ class ServerManagerJob implements ShouldQueue
 
     private function getServers(): Collection
     {
-        $allServers = Server::where('ip', '!=', '1.2.3.4');
+        $allServers = Server::with('settings')->where('ip', '!=', '1.2.3.4');
 
         if (isCloud()) {
             $servers = $allServers->whereRelation('team.subscription', 'stripe_invoice_paid', true)->get();
-            $own = Team::find(0)->servers;
+            $own = Team::find(0)->servers()->with('settings')->get();
 
             return $servers->merge($own);
         } else {
@@ -82,6 +82,10 @@ class ServerManagerJob implements ShouldQueue
         if ($this->shouldRunNow($this->checkFrequency)) {
             $servers->each(function (Server $server) {
                 try {
+                    // Skip SSH connection check if Sentinel is healthy — its heartbeat already proves connectivity
+                    if ($server->isSentinelEnabled() && $server->isSentinelLive()) {
+                        return;
+                    }
                     ServerConnectionCheckJob::dispatch($server);
                 } catch (\Exception $e) {
                     Log::channel('scheduled-errors')->error('Failed to dispatch ServerConnectionCheck', [
@@ -134,9 +138,7 @@ class ServerManagerJob implements ShouldQueue
         // Dispatch Sentinel restart if due (daily for Sentinel-enabled servers)
 
         if ($shouldRestartSentinel) {
-            dispatch(function () use ($server) {
-                $server->restartContainer('coolify-sentinel');
-            });
+            CheckAndStartSentinelJob::dispatch($server);
         }
 
         // Dispatch ServerStorageCheckJob if due (only when Sentinel is out of sync or disabled)
@@ -160,11 +162,8 @@ class ServerManagerJob implements ShouldQueue
             ServerPatchCheckJob::dispatch($server);
         }
 
-        // Sentinel update checks (hourly) - check for updates to Sentinel version
-        // No timezone needed for hourly - runs at top of every hour
-        if ($isSentinelEnabled && $this->shouldRunNow('0 * * * *')) {
-            CheckAndStartSentinelJob::dispatch($server);
-        }
+        // Note: CheckAndStartSentinelJob is only dispatched daily (line above) for version updates.
+        // Crash recovery is handled by sentinelOutOfSync → ServerCheckJob → CheckAndStartSentinelJob.
     }
 
     private function shouldRunNow(string $frequency, ?string $timezone = null): bool