feat(jobs): add queue delay resilience to scheduled job execution

Implement dedup key-based cron tracking to make scheduled jobs resilient to queue
delays. Even if a job is delayed by minutes, it will catch the missed cron window
by tracking previousRunDate in cache instead of relying on isDue() alone.

- Add dedupKey parameter to shouldRunNow() in ScheduledJobManager
  - When provided, uses getPreviousRunDate() + cache tracking for resilience
  - Falls back to isDue() for docker cleanups without dedup key
  - Prevents double-dispatch within same cron window

- Optimize ServerConnectionCheckJob dispatch
  - Skip SSH checks if Sentinel is healthy (enabled and live)
  - Reduces redundant checks when Sentinel heartbeat proves connectivity

- Remove hourly Sentinel update checks
  - Consolidate to daily CheckAndStartSentinelJob dispatch
  - Crash recovery handled by sentinelOutOfSync → ServerCheckJob flow

- Add logging for skipped database backups with context (backup_id, database_id, status)

- Refactor skip reason methods to accept server parameter, avoiding redundant queries

- Add comprehensive test suite for scheduling with various delay scenarios and timezones

tests/Feature/ScheduledJobManagerShouldRunNowTest.php

@@ -0,0 +1,208 @@
+<?php
+
+use App\Jobs\ScheduledJobManager;
+use Illuminate\Support\Carbon;
+use Illuminate\Support\Facades\Cache;
+
+beforeEach(function () {
+    // Clear any dedup keys
+    Cache::flush();
+});
+
+it('dispatches backup when job runs on time at the cron minute', function () {
+    // Freeze time at exactly 02:00 — daily cron "0 2 * * *" is due
+    Carbon::setTestNow(Carbon::create(2026, 2, 28, 2, 0, 0, 'UTC'));
+
+    $job = new ScheduledJobManager;
+
+    // Use reflection to test shouldRunNow
+    $reflection = new ReflectionClass($job);
+
+    $executionTimeProp = $reflection->getProperty('executionTime');
+    $executionTimeProp->setAccessible(true);
+    $executionTimeProp->setValue($job, Carbon::now());
+
+    $method = $reflection->getMethod('shouldRunNow');
+    $method->setAccessible(true);
+
+    $result = $method->invoke($job, '0 2 * * *', 'UTC', 'test-backup:1');
+
+    expect($result)->toBeTrue();
+});
+
+it('dispatches backup when job is delayed past the cron minute', function () {
+    // Freeze time at 02:07 — job was delayed 7 minutes past the 02:00 cron
+    Carbon::setTestNow(Carbon::create(2026, 2, 28, 2, 7, 0, 'UTC'));
+
+    $job = new ScheduledJobManager;
+
+    $reflection = new ReflectionClass($job);
+
+    $executionTimeProp = $reflection->getProperty('executionTime');
+    $executionTimeProp->setAccessible(true);
+    $executionTimeProp->setValue($job, Carbon::now());
+
+    $method = $reflection->getMethod('shouldRunNow');
+    $method->setAccessible(true);
+
+    // isDue() would return false at 02:07, but getPreviousRunDate() = 02:00
+    // No lastDispatched in cache → should run
+    $result = $method->invoke($job, '0 2 * * *', 'UTC', 'test-backup:1');
+
+    expect($result)->toBeTrue();
+});
+
+it('does not double-dispatch on subsequent runs within same cron window', function () {
+    // First run at 02:00 — dispatches and sets cache
+    Carbon::setTestNow(Carbon::create(2026, 2, 28, 2, 0, 0, 'UTC'));
+
+    $job = new ScheduledJobManager;
+    $reflection = new ReflectionClass($job);
+
+    $executionTimeProp = $reflection->getProperty('executionTime');
+    $executionTimeProp->setAccessible(true);
+    $executionTimeProp->setValue($job, Carbon::now());
+
+    $method = $reflection->getMethod('shouldRunNow');
+    $method->setAccessible(true);
+
+    $first = $method->invoke($job, '0 2 * * *', 'UTC', 'test-backup:2');
+    expect($first)->toBeTrue();
+
+    // Second run at 02:01 — should NOT dispatch (previousDue=02:00, lastDispatched=02:00)
+    Carbon::setTestNow(Carbon::create(2026, 2, 28, 2, 1, 0, 'UTC'));
+    $executionTimeProp->setValue($job, Carbon::now());
+
+    $second = $method->invoke($job, '0 2 * * *', 'UTC', 'test-backup:2');
+    expect($second)->toBeFalse();
+});
+
+it('fires every_minute cron correctly on consecutive minutes', function () {
+    $job = new ScheduledJobManager;
+    $reflection = new ReflectionClass($job);
+
+    $executionTimeProp = $reflection->getProperty('executionTime');
+    $executionTimeProp->setAccessible(true);
+
+    $method = $reflection->getMethod('shouldRunNow');
+    $method->setAccessible(true);
+
+    // Minute 1
+    Carbon::setTestNow(Carbon::create(2026, 2, 28, 10, 0, 0, 'UTC'));
+    $executionTimeProp->setValue($job, Carbon::now());
+    $result1 = $method->invoke($job, '* * * * *', 'UTC', 'test-backup:3');
+    expect($result1)->toBeTrue();
+
+    // Minute 2
+    Carbon::setTestNow(Carbon::create(2026, 2, 28, 10, 1, 0, 'UTC'));
+    $executionTimeProp->setValue($job, Carbon::now());
+    $result2 = $method->invoke($job, '* * * * *', 'UTC', 'test-backup:3');
+    expect($result2)->toBeTrue();
+
+    // Minute 3
+    Carbon::setTestNow(Carbon::create(2026, 2, 28, 10, 2, 0, 'UTC'));
+    $executionTimeProp->setValue($job, Carbon::now());
+    $result3 = $method->invoke($job, '* * * * *', 'UTC', 'test-backup:3');
+    expect($result3)->toBeTrue();
+});
+
+it('re-dispatches after cache loss instead of missing', function () {
+    // First run at 02:00 — dispatches
+    Carbon::setTestNow(Carbon::create(2026, 2, 28, 2, 0, 0, 'UTC'));
+
+    $job = new ScheduledJobManager;
+    $reflection = new ReflectionClass($job);
+
+    $executionTimeProp = $reflection->getProperty('executionTime');
+    $executionTimeProp->setAccessible(true);
+    $executionTimeProp->setValue($job, Carbon::now());
+
+    $method = $reflection->getMethod('shouldRunNow');
+    $method->setAccessible(true);
+
+    $method->invoke($job, '0 2 * * *', 'UTC', 'test-backup:4');
+
+    // Simulate Redis restart — cache lost
+    Cache::forget('test-backup:4');
+
+    // Run again at 02:01 — should re-dispatch (lastDispatched = null after cache loss)
+    Carbon::setTestNow(Carbon::create(2026, 2, 28, 2, 1, 0, 'UTC'));
+    $executionTimeProp->setValue($job, Carbon::now());
+
+    $result = $method->invoke($job, '0 2 * * *', 'UTC', 'test-backup:4');
+    expect($result)->toBeTrue();
+});
+
+it('does not dispatch when cron is not due and was not recently due', function () {
+    // Time is 10:00, cron is daily at 02:00 — last due was 8 hours ago
+    Carbon::setTestNow(Carbon::create(2026, 2, 28, 10, 0, 0, 'UTC'));
+
+    $job = new ScheduledJobManager;
+    $reflection = new ReflectionClass($job);
+
+    $executionTimeProp = $reflection->getProperty('executionTime');
+    $executionTimeProp->setAccessible(true);
+    $executionTimeProp->setValue($job, Carbon::now());
+
+    $method = $reflection->getMethod('shouldRunNow');
+    $method->setAccessible(true);
+
+    // previousDue = 02:00, but lastDispatched was set at 02:00 (simulate)
+    Cache::put('test-backup:5', Carbon::create(2026, 2, 28, 2, 0, 0, 'UTC')->toIso8601String(), 86400);
+
+    $result = $method->invoke($job, '0 2 * * *', 'UTC', 'test-backup:5');
+    expect($result)->toBeFalse();
+});
+
+it('falls back to isDue when no dedup key is provided', function () {
+    // Time is exactly 02:00, cron is "0 2 * * *" — should be due
+    Carbon::setTestNow(Carbon::create(2026, 2, 28, 2, 0, 0, 'UTC'));
+
+    $job = new ScheduledJobManager;
+    $reflection = new ReflectionClass($job);
+
+    $executionTimeProp = $reflection->getProperty('executionTime');
+    $executionTimeProp->setAccessible(true);
+    $executionTimeProp->setValue($job, Carbon::now());
+
+    $method = $reflection->getMethod('shouldRunNow');
+    $method->setAccessible(true);
+
+    // No dedup key → simple isDue check
+    $result = $method->invoke($job, '0 2 * * *', 'UTC');
+    expect($result)->toBeTrue();
+
+    // At 02:01 without dedup key → isDue returns false
+    Carbon::setTestNow(Carbon::create(2026, 2, 28, 2, 1, 0, 'UTC'));
+    $executionTimeProp->setValue($job, Carbon::now());
+
+    $result2 = $method->invoke($job, '0 2 * * *', 'UTC');
+    expect($result2)->toBeFalse();
+});
+
+it('respects server timezone for cron evaluation', function () {
+    // UTC time is 22:00 Feb 28, which is 06:00 Mar 1 in Asia/Singapore (+8)
+    Carbon::setTestNow(Carbon::create(2026, 2, 28, 22, 0, 0, 'UTC'));
+
+    $job = new ScheduledJobManager;
+    $reflection = new ReflectionClass($job);
+
+    $executionTimeProp = $reflection->getProperty('executionTime');
+    $executionTimeProp->setAccessible(true);
+    $executionTimeProp->setValue($job, Carbon::now());
+
+    $method = $reflection->getMethod('shouldRunNow');
+    $method->setAccessible(true);
+
+    // Simulate that today's 06:00 UTC run was already dispatched (at 06:00 UTC)
+    Cache::put('test-backup:7', Carbon::create(2026, 2, 28, 6, 0, 0, 'UTC')->toIso8601String(), 86400);
+
+    // Cron "0 6 * * *" in Asia/Singapore: local time is 06:00 Mar 1 → previousDue = 06:00 Mar 1 SGT
+    // That's a NEW cron window (Mar 1) that hasn't been dispatched → should fire
+    $resultSingapore = $method->invoke($job, '0 6 * * *', 'Asia/Singapore', 'test-backup:6');
+    expect($resultSingapore)->toBeTrue();
+
+    // Cron "0 6 * * *" in UTC: previousDue = 06:00 Feb 28 UTC, already dispatched at 06:00 → should NOT fire
+    $resultUtc = $method->invoke($job, '0 6 * * *', 'UTC', 'test-backup:7');
+    expect($resultUtc)->toBeFalse();
+});

tests/Unit/ServerManagerJobSentinelCheckTest.php

@@ -1,6 +1,7 @@
 <?php
 
 use App\Jobs\CheckAndStartSentinelJob;
+use App\Jobs\ServerConnectionCheckJob;
 use App\Jobs\ServerManagerJob;
 use App\Models\InstanceSettings;
 use App\Models\Server;
@@ -10,23 +11,22 @@ use Mockery;
 
 beforeEach(function () {
     Queue::fake();
-    Carbon::setTestNow('2025-01-15 12:00:00'); // Set to top of the hour for cron matching
+    Carbon::setTestNow('2025-01-15 12:00:00');
 });
 
 afterEach(function () {
     Mockery::close();
-    Carbon::setTestNow(); // Reset frozen time
+    Carbon::setTestNow();
 });
 
-it('dispatches CheckAndStartSentinelJob hourly for sentinel-enabled servers', function () {
-    // Mock InstanceSettings
+it('does not dispatch CheckAndStartSentinelJob hourly anymore', function () {
     $settings = Mockery::mock(InstanceSettings::class);
     $settings->instance_timezone = 'UTC';
     $this->app->instance(InstanceSettings::class, $settings);
 
-    // Create a mock server with sentinel enabled
     $server = Mockery::mock(Server::class)->makePartial();
     $server->shouldReceive('isSentinelEnabled')->andReturn(true);
+    $server->shouldReceive('isSentinelLive')->andReturn(true);
     $server->id = 1;
     $server->name = 'test-server';
     $server->ip = '192.168.1.100';
@@ -34,29 +34,76 @@ it('dispatches CheckAndStartSentinelJob hourly for sentinel-enabled servers', fu
     $server->shouldReceive('getAttribute')->with('settings')->andReturn((object) ['server_timezone' => 'UTC']);
     $server->shouldReceive('waitBeforeDoingSshCheck')->andReturn(120);
 
-    // Mock the Server query
     Server::shouldReceive('where')->with('ip', '!=', '1.2.3.4')->andReturnSelf();
     Server::shouldReceive('get')->andReturn(collect([$server]));
 
-    // Execute the job
     $job = new ServerManagerJob;
     $job->handle();
 
-    // Assert CheckAndStartSentinelJob was dispatched for the sentinel-enabled server
-    Queue::assertPushed(CheckAndStartSentinelJob::class, function ($job) use ($server) {
-        return $job->server->id === $server->id;
-    });
+    // Hourly CheckAndStartSentinelJob dispatch was removed — ServerCheckJob handles it when Sentinel is out of sync
+    Queue::assertNotPushed(CheckAndStartSentinelJob::class);
 });
 
-it('does not dispatch CheckAndStartSentinelJob for servers without sentinel enabled', function () {
-    // Mock InstanceSettings
+it('skips ServerConnectionCheckJob when sentinel is live', function () {
+    $settings = Mockery::mock(InstanceSettings::class);
+    $settings->instance_timezone = 'UTC';
+    $this->app->instance(InstanceSettings::class, $settings);
+
+    $server = Mockery::mock(Server::class)->makePartial();
+    $server->shouldReceive('isSentinelEnabled')->andReturn(true);
+    $server->shouldReceive('isSentinelLive')->andReturn(true);
+    $server->id = 1;
+    $server->name = 'test-server';
+    $server->ip = '192.168.1.100';
+    $server->sentinel_updated_at = Carbon::now();
+    $server->shouldReceive('getAttribute')->with('settings')->andReturn((object) ['server_timezone' => 'UTC']);
+    $server->shouldReceive('waitBeforeDoingSshCheck')->andReturn(120);
+
+    Server::shouldReceive('where')->with('ip', '!=', '1.2.3.4')->andReturnSelf();
+    Server::shouldReceive('get')->andReturn(collect([$server]));
+
+    $job = new ServerManagerJob;
+    $job->handle();
+
+    // Sentinel is healthy so SSH connection check is skipped
+    Queue::assertNotPushed(ServerConnectionCheckJob::class);
+});
+
+it('dispatches ServerConnectionCheckJob when sentinel is not live', function () {
+    $settings = Mockery::mock(InstanceSettings::class);
+    $settings->instance_timezone = 'UTC';
+    $this->app->instance(InstanceSettings::class, $settings);
+
+    $server = Mockery::mock(Server::class)->makePartial();
+    $server->shouldReceive('isSentinelEnabled')->andReturn(true);
+    $server->shouldReceive('isSentinelLive')->andReturn(false);
+    $server->id = 1;
+    $server->name = 'test-server';
+    $server->ip = '192.168.1.100';
+    $server->sentinel_updated_at = Carbon::now()->subMinutes(10);
+    $server->shouldReceive('getAttribute')->with('settings')->andReturn((object) ['server_timezone' => 'UTC']);
+    $server->shouldReceive('waitBeforeDoingSshCheck')->andReturn(120);
+
+    Server::shouldReceive('where')->with('ip', '!=', '1.2.3.4')->andReturnSelf();
+    Server::shouldReceive('get')->andReturn(collect([$server]));
+
+    $job = new ServerManagerJob;
+    $job->handle();
+
+    // Sentinel is out of sync so SSH connection check is needed
+    Queue::assertPushed(ServerConnectionCheckJob::class, function ($job) use ($server) {
+        return $job->server->id === $server->id;
+    });
+});
+
+it('dispatches ServerConnectionCheckJob when sentinel is not enabled', function () {
     $settings = Mockery::mock(InstanceSettings::class);
     $settings->instance_timezone = 'UTC';
     $this->app->instance(InstanceSettings::class, $settings);
 
-    // Create a mock server with sentinel disabled
     $server = Mockery::mock(Server::class)->makePartial();
     $server->shouldReceive('isSentinelEnabled')->andReturn(false);
+    $server->shouldReceive('isSentinelLive')->never();
     $server->id = 2;
     $server->name = 'test-server-no-sentinel';
     $server->ip = '192.168.1.101';
@@ -64,78 +111,14 @@ it('does not dispatch CheckAndStartSentinelJob for servers without sentinel enab
     $server->shouldReceive('getAttribute')->with('settings')->andReturn((object) ['server_timezone' => 'UTC']);
     $server->shouldReceive('waitBeforeDoingSshCheck')->andReturn(120);
 
-    // Mock the Server query
     Server::shouldReceive('where')->with('ip', '!=', '1.2.3.4')->andReturnSelf();
     Server::shouldReceive('get')->andReturn(collect([$server]));
 
-    // Execute the job
     $job = new ServerManagerJob;
     $job->handle();
 
-    // Assert CheckAndStartSentinelJob was NOT dispatched
-    Queue::assertNotPushed(CheckAndStartSentinelJob::class);
-});
-
-it('respects server timezone when scheduling sentinel checks', function () {
-    // Mock InstanceSettings
-    $settings = Mockery::mock(InstanceSettings::class);
-    $settings->instance_timezone = 'UTC';
-    $this->app->instance(InstanceSettings::class, $settings);
-
-    // Set test time to top of hour in America/New_York (which is 17:00 UTC)
-    Carbon::setTestNow('2025-01-15 17:00:00'); // 12:00 PM EST (top of hour in EST)
-
-    // Create a mock server with sentinel enabled and America/New_York timezone
-    $server = Mockery::mock(Server::class)->makePartial();
-    $server->shouldReceive('isSentinelEnabled')->andReturn(true);
-    $server->id = 3;
-    $server->name = 'test-server-est';
-    $server->ip = '192.168.1.102';
-    $server->sentinel_updated_at = Carbon::now();
-    $server->shouldReceive('getAttribute')->with('settings')->andReturn((object) ['server_timezone' => 'America/New_York']);
-    $server->shouldReceive('waitBeforeDoingSshCheck')->andReturn(120);
-
-    // Mock the Server query
-    Server::shouldReceive('where')->with('ip', '!=', '1.2.3.4')->andReturnSelf();
-    Server::shouldReceive('get')->andReturn(collect([$server]));
-
-    // Execute the job
-    $job = new ServerManagerJob;
-    $job->handle();
-
-    // Assert CheckAndStartSentinelJob was dispatched (should run at top of hour in server's timezone)
-    Queue::assertPushed(CheckAndStartSentinelJob::class, function ($job) use ($server) {
+    // Sentinel is not enabled so SSH connection check must run
+    Queue::assertPushed(ServerConnectionCheckJob::class, function ($job) use ($server) {
         return $job->server->id === $server->id;
     });
 });
-
-it('does not dispatch sentinel check when not at top of hour', function () {
-    // Mock InstanceSettings
-    $settings = Mockery::mock(InstanceSettings::class);
-    $settings->instance_timezone = 'UTC';
-    $this->app->instance(InstanceSettings::class, $settings);
-
-    // Set test time to middle of the hour (not top of hour)
-    Carbon::setTestNow('2025-01-15 12:30:00');
-
-    // Create a mock server with sentinel enabled
-    $server = Mockery::mock(Server::class)->makePartial();
-    $server->shouldReceive('isSentinelEnabled')->andReturn(true);
-    $server->id = 4;
-    $server->name = 'test-server-mid-hour';
-    $server->ip = '192.168.1.103';
-    $server->sentinel_updated_at = Carbon::now();
-    $server->shouldReceive('getAttribute')->with('settings')->andReturn((object) ['server_timezone' => 'UTC']);
-    $server->shouldReceive('waitBeforeDoingSshCheck')->andReturn(120);
-
-    // Mock the Server query
-    Server::shouldReceive('where')->with('ip', '!=', '1.2.3.4')->andReturnSelf();
-    Server::shouldReceive('get')->andReturn(collect([$server]));
-
-    // Execute the job
-    $job = new ServerManagerJob;
-    $job->handle();
-
-    // Assert CheckAndStartSentinelJob was NOT dispatched (not top of hour)
-    Queue::assertNotPushed(CheckAndStartSentinelJob::class);
-});