refactor(scheduler): extract cron scheduling logic to shared helper

Extract the shouldRunNow() method from ScheduledJobManager and ServerManagerJob into
a reusable shouldRunCronNow() helper function. This centralizes cron scheduling logic
and enables consistent deduplication behavior across all scheduled job types.

- Create shouldRunCronNow() helper in bootstrap/helpers/shared.php with timezone
  and dedup support
- Refactor ScheduledJobManager and ServerManagerJob to use the shared helper
- Add ScheduledJobDiagnostics command for inspecting cache state and scheduling
  decisions across all scheduled jobs
- Simplify shouldRunNow tests to directly test the helper function
- Add DockerCleanupJob test for error handling and execution tracking
- Increase scheduled log retention from 1 to 7 days

app/Console/Commands/ScheduledJobDiagnostics.php

@@ -0,0 +1,255 @@
+<?php
+
+namespace App\Console\Commands;
+
+use App\Models\DockerCleanupExecution;
+use App\Models\ScheduledDatabaseBackup;
+use App\Models\ScheduledTask;
+use App\Models\Server;
+use App\Models\Team;
+use Illuminate\Console\Command;
+use Illuminate\Support\Carbon;
+use Illuminate\Support\Facades\Cache;
+
+class ScheduledJobDiagnostics extends Command
+{
+    protected $signature = 'scheduled:diagnostics
+        {--type=all : Type to inspect: docker-cleanup, backups, tasks, server-jobs, all}
+        {--server= : Filter by server ID}';
+
+    protected $description = 'Inspect dedup cache state and scheduling decisions for all scheduled jobs';
+
+    public function handle(): int
+    {
+        $type = $this->option('type');
+        $serverFilter = $this->option('server');
+
+        $this->outputHeartbeat();
+
+        if (in_array($type, ['all', 'docker-cleanup'])) {
+            $this->inspectDockerCleanups($serverFilter);
+        }
+
+        if (in_array($type, ['all', 'backups'])) {
+            $this->inspectBackups();
+        }
+
+        if (in_array($type, ['all', 'tasks'])) {
+            $this->inspectTasks();
+        }
+
+        if (in_array($type, ['all', 'server-jobs'])) {
+            $this->inspectServerJobs($serverFilter);
+        }
+
+        return self::SUCCESS;
+    }
+
+    private function outputHeartbeat(): void
+    {
+        $heartbeat = Cache::get('scheduled-job-manager:heartbeat');
+        if ($heartbeat) {
+            $age = Carbon::parse($heartbeat)->diffForHumans();
+            $this->info("Scheduler heartbeat: {$heartbeat} ({$age})");
+        } else {
+            $this->error('Scheduler heartbeat: MISSING — ScheduledJobManager may not be running');
+        }
+        $this->newLine();
+    }
+
+    private function inspectDockerCleanups(?string $serverFilter): void
+    {
+        $this->info('=== Docker Cleanup Jobs ===');
+
+        $servers = $this->getServers($serverFilter);
+
+        $rows = [];
+        foreach ($servers as $server) {
+            $frequency = data_get($server->settings, 'docker_cleanup_frequency', '0 * * * *');
+            if (isset(VALID_CRON_STRINGS[$frequency])) {
+                $frequency = VALID_CRON_STRINGS[$frequency];
+            }
+
+            $dedupKey = "docker-cleanup:{$server->id}";
+            $cacheValue = Cache::get($dedupKey);
+            $timezone = data_get($server->settings, 'server_timezone', config('app.timezone'));
+
+            if (validate_timezone($timezone) === false) {
+                $timezone = config('app.timezone');
+            }
+
+            $wouldFire = shouldRunCronNow($frequency, $timezone, $dedupKey);
+
+            $lastExecution = DockerCleanupExecution::where('server_id', $server->id)
+                ->latest()
+                ->first();
+
+            $rows[] = [
+                $server->id,
+                $server->name,
+                $timezone,
+                $frequency,
+                $dedupKey,
+                $cacheValue ?? '<missing>',
+                $wouldFire ? 'YES' : 'no',
+                $lastExecution ? $lastExecution->status.' @ '.$lastExecution->created_at : 'never',
+            ];
+        }
+
+        $this->table(
+            ['ID', 'Server', 'TZ', 'Frequency', 'Dedup Key', 'Cache Value', 'Would Fire', 'Last Execution'],
+            $rows
+        );
+        $this->newLine();
+    }
+
+    private function inspectBackups(): void
+    {
+        $this->info('=== Scheduled Backups ===');
+
+        $backups = ScheduledDatabaseBackup::with(['database'])
+            ->where('enabled', true)
+            ->get();
+
+        $rows = [];
+        foreach ($backups as $backup) {
+            $server = $backup->server();
+            $frequency = $backup->frequency;
+            if (isset(VALID_CRON_STRINGS[$frequency])) {
+                $frequency = VALID_CRON_STRINGS[$frequency];
+            }
+
+            $dedupKey = "scheduled-backup:{$backup->id}";
+            $cacheValue = Cache::get($dedupKey);
+            $timezone = $server ? data_get($server->settings, 'server_timezone', config('app.timezone')) : config('app.timezone');
+
+            if (validate_timezone($timezone) === false) {
+                $timezone = config('app.timezone');
+            }
+
+            $wouldFire = shouldRunCronNow($frequency, $timezone, $dedupKey);
+
+            $rows[] = [
+                $backup->id,
+                $backup->database_type ?? 'unknown',
+                $server?->name ?? 'N/A',
+                $frequency,
+                $cacheValue ?? '<missing>',
+                $wouldFire ? 'YES' : 'no',
+            ];
+        }
+
+        $this->table(
+            ['Backup ID', 'DB Type', 'Server', 'Frequency', 'Cache Value', 'Would Fire'],
+            $rows
+        );
+        $this->newLine();
+    }
+
+    private function inspectTasks(): void
+    {
+        $this->info('=== Scheduled Tasks ===');
+
+        $tasks = ScheduledTask::with(['service', 'application'])
+            ->where('enabled', true)
+            ->get();
+
+        $rows = [];
+        foreach ($tasks as $task) {
+            $server = $task->server();
+            $frequency = $task->frequency;
+            if (isset(VALID_CRON_STRINGS[$frequency])) {
+                $frequency = VALID_CRON_STRINGS[$frequency];
+            }
+
+            $dedupKey = "scheduled-task:{$task->id}";
+            $cacheValue = Cache::get($dedupKey);
+            $timezone = $server ? data_get($server->settings, 'server_timezone', config('app.timezone')) : config('app.timezone');
+
+            if (validate_timezone($timezone) === false) {
+                $timezone = config('app.timezone');
+            }
+
+            $wouldFire = shouldRunCronNow($frequency, $timezone, $dedupKey);
+
+            $rows[] = [
+                $task->id,
+                $task->name,
+                $server?->name ?? 'N/A',
+                $frequency,
+                $cacheValue ?? '<missing>',
+                $wouldFire ? 'YES' : 'no',
+            ];
+        }
+
+        $this->table(
+            ['Task ID', 'Name', 'Server', 'Frequency', 'Cache Value', 'Would Fire'],
+            $rows
+        );
+        $this->newLine();
+    }
+
+    private function inspectServerJobs(?string $serverFilter): void
+    {
+        $this->info('=== Server Manager Jobs ===');
+
+        $servers = $this->getServers($serverFilter);
+
+        $rows = [];
+        foreach ($servers as $server) {
+            $timezone = data_get($server->settings, 'server_timezone', config('app.timezone'));
+            if (validate_timezone($timezone) === false) {
+                $timezone = config('app.timezone');
+            }
+
+            $dedupKeys = [
+                "sentinel-restart:{$server->id}" => '0 0 * * *',
+                "server-patch-check:{$server->id}" => '0 0 * * 0',
+                "server-check:{$server->id}" => isCloud() ? '*/5 * * * *' : '* * * * *',
+                "server-storage-check:{$server->id}" => data_get($server->settings, 'server_disk_usage_check_frequency', '0 23 * * *'),
+            ];
+
+            foreach ($dedupKeys as $dedupKey => $frequency) {
+                if (isset(VALID_CRON_STRINGS[$frequency])) {
+                    $frequency = VALID_CRON_STRINGS[$frequency];
+                }
+
+                $cacheValue = Cache::get($dedupKey);
+                $wouldFire = shouldRunCronNow($frequency, $timezone, $dedupKey);
+
+                $rows[] = [
+                    $server->id,
+                    $server->name,
+                    $dedupKey,
+                    $frequency,
+                    $cacheValue ?? '<missing>',
+                    $wouldFire ? 'YES' : 'no',
+                ];
+            }
+        }
+
+        $this->table(
+            ['Server ID', 'Server', 'Dedup Key', 'Frequency', 'Cache Value', 'Would Fire'],
+            $rows
+        );
+        $this->newLine();
+    }
+
+    private function getServers(?string $serverFilter): \Illuminate\Support\Collection
+    {
+        $query = Server::with('settings')->where('ip', '!=', '1.2.3.4');
+
+        if ($serverFilter) {
+            $query->where('id', $serverFilter);
+        }
+
+        if (isCloud()) {
+            $servers = $query->whereRelation('team.subscription', 'stripe_invoice_paid', true)->get();
+            $own = Team::find(0)?->servers()->with('settings')->get() ?? collect();
+
+            return $servers->merge($own);
+        }
+
+        return $query->get();
+    }
+}

app/Jobs/ScheduledJobManager.php

@@ -6,7 +6,6 @@ use App\Models\ScheduledDatabaseBackup;
 use App\Models\ScheduledTask;
 use App\Models\Server;
 use App\Models\Team;
-use Cron\CronExpression;
 use Illuminate\Bus\Queueable;
 use Illuminate\Contracts\Queue\ShouldQueue;
 use Illuminate\Foundation\Bus\Dispatchable;
@@ -185,7 +184,7 @@ class ScheduledJobManager implements ShouldQueue
                     $frequency = VALID_CRON_STRINGS[$frequency];
                 }
 
-                if ($this->shouldRunNow($frequency, $serverTimezone, "scheduled-backup:{$backup->id}")) {
+                if (shouldRunCronNow($frequency, $serverTimezone, "scheduled-backup:{$backup->id}", $this->executionTime)) {
                     DatabaseBackupJob::dispatch($backup);
                     $this->dispatchedCount++;
                     Log::channel('scheduled')->info('Backup dispatched', [
@@ -239,7 +238,7 @@ class ScheduledJobManager implements ShouldQueue
                     $frequency = VALID_CRON_STRINGS[$frequency];
                 }
 
-                if (! $this->shouldRunNow($frequency, $serverTimezone, "scheduled-task:{$task->id}")) {
+                if (! shouldRunCronNow($frequency, $serverTimezone, "scheduled-task:{$task->id}", $this->executionTime)) {
                     continue;
                 }
 
@@ -336,51 +335,6 @@ class ScheduledJobManager implements ShouldQueue
         return null;
     }
 
-    /**
-     * Determine if a cron schedule should run now.
-     *
-     * When a dedupKey is provided, uses getPreviousRunDate() + last-dispatch tracking
-     * instead of isDue(). This is resilient to queue delays — even if the job is delayed
-     * by minutes, it still catches the missed cron window. Without dedupKey, falls back
-     * to simple isDue() check.
-     */
-    private function shouldRunNow(string $frequency, string $timezone, ?string $dedupKey = null): bool
-    {
-        $cron = new CronExpression($frequency);
-        $baseTime = $this->executionTime ?? Carbon::now();
-        $executionTime = $baseTime->copy()->setTimezone($timezone);
-
-        // No dedup key → simple isDue check
-        if ($dedupKey === null) {
-            return $cron->isDue($executionTime);
-        }
-
-        // Get the most recent time this cron was due (including current minute)
-        $previousDue = Carbon::instance($cron->getPreviousRunDate($executionTime, allowCurrentDate: true));
-
-        $lastDispatched = Cache::get($dedupKey);
-
-        if ($lastDispatched === null) {
-            // First run after restart or cache loss: only fire if actually due right now.
-            // Seed the cache so subsequent runs can use tolerance/catch-up logic.
-            $isDue = $cron->isDue($executionTime);
-            if ($isDue) {
-                Cache::put($dedupKey, $executionTime->toIso8601String(), 86400);
-            }
-
-            return $isDue;
-        }
-
-        // Subsequent runs: fire if there's been a due time since last dispatch
-        if ($previousDue->gt(Carbon::parse($lastDispatched))) {
-            Cache::put($dedupKey, $executionTime->toIso8601String(), 86400);
-
-            return true;
-        }
-
-        return false;
-    }
-
     private function processDockerCleanups(): void
     {
         // Get all servers that need cleanup checks
@@ -411,7 +365,7 @@ class ScheduledJobManager implements ShouldQueue
                 }
 
                 // Use the frozen execution time for consistent evaluation
-                if ($this->shouldRunNow($frequency, $serverTimezone, "docker-cleanup:{$server->id}")) {
+                if (shouldRunCronNow($frequency, $serverTimezone, "docker-cleanup:{$server->id}", $this->executionTime)) {
                     DockerCleanupJob::dispatch(
                         $server,
                         false,

app/Jobs/ServerManagerJob.php

@@ -5,7 +5,6 @@ namespace App\Jobs;
 use App\Models\InstanceSettings;
 use App\Models\Server;
 use App\Models\Team;
-use Cron\CronExpression;
 use Illuminate\Bus\Queueable;
 use Illuminate\Contracts\Queue\ShouldBeEncrypted;
 use Illuminate\Contracts\Queue\ShouldQueue;
@@ -14,7 +13,6 @@ use Illuminate\Queue\InteractsWithQueue;
 use Illuminate\Queue\SerializesModels;
 use Illuminate\Support\Carbon;
 use Illuminate\Support\Collection;
-use Illuminate\Support\Facades\Cache;
 use Illuminate\Support\Facades\Log;
 
 class ServerManagerJob implements ShouldBeEncrypted, ShouldQueue
@@ -81,7 +79,7 @@ class ServerManagerJob implements ShouldBeEncrypted, ShouldQueue
     private function dispatchConnectionChecks(Collection $servers): void
     {
 
-        if ($this->shouldRunNow($this->checkFrequency, dedupKey: 'server-connection-checks')) {
+        if (shouldRunCronNow($this->checkFrequency, $this->instanceTimezone, 'server-connection-checks', $this->executionTime)) {
             $servers->each(function (Server $server) {
                 try {
                     // Skip SSH connection check if Sentinel is healthy — its heartbeat already proves connectivity
@@ -130,13 +128,13 @@ class ServerManagerJob implements ShouldBeEncrypted, ShouldQueue
 
         if ($sentinelOutOfSync) {
             // Dispatch ServerCheckJob if Sentinel is out of sync
-            if ($this->shouldRunNow($this->checkFrequency, $serverTimezone, "server-check:{$server->id}")) {
+            if (shouldRunCronNow($this->checkFrequency, $serverTimezone, "server-check:{$server->id}", $this->executionTime)) {
                 ServerCheckJob::dispatch($server);
             }
         }
 
         $isSentinelEnabled = $server->isSentinelEnabled();
-        $shouldRestartSentinel = $isSentinelEnabled && $this->shouldRunNow('0 0 * * *', $serverTimezone, "sentinel-restart:{$server->id}");
+        $shouldRestartSentinel = $isSentinelEnabled && shouldRunCronNow('0 0 * * *', $serverTimezone, "sentinel-restart:{$server->id}", $this->executionTime);
         // Dispatch Sentinel restart if due (daily for Sentinel-enabled servers)
 
         if ($shouldRestartSentinel) {
@@ -150,7 +148,7 @@ class ServerManagerJob implements ShouldBeEncrypted, ShouldQueue
             if (isset(VALID_CRON_STRINGS[$serverDiskUsageCheckFrequency])) {
                 $serverDiskUsageCheckFrequency = VALID_CRON_STRINGS[$serverDiskUsageCheckFrequency];
             }
-            $shouldRunStorageCheck = $this->shouldRunNow($serverDiskUsageCheckFrequency, $serverTimezone, "server-storage-check:{$server->id}");
+            $shouldRunStorageCheck = shouldRunCronNow($serverDiskUsageCheckFrequency, $serverTimezone, "server-storage-check:{$server->id}", $this->executionTime);
 
             if ($shouldRunStorageCheck) {
                 ServerStorageCheckJob::dispatch($server);
@@ -158,7 +156,7 @@ class ServerManagerJob implements ShouldBeEncrypted, ShouldQueue
         }
 
         // Dispatch ServerPatchCheckJob if due (weekly)
-        $shouldRunPatchCheck = $this->shouldRunNow('0 0 * * 0', $serverTimezone, "server-patch-check:{$server->id}");
+        $shouldRunPatchCheck = shouldRunCronNow('0 0 * * 0', $serverTimezone, "server-patch-check:{$server->id}", $this->executionTime);
 
         if ($shouldRunPatchCheck) { // Weekly on Sunday at midnight
             ServerPatchCheckJob::dispatch($server);
@@ -167,45 +165,4 @@ class ServerManagerJob implements ShouldBeEncrypted, ShouldQueue
         // Note: CheckAndStartSentinelJob is only dispatched daily (line above) for version updates.
         // Crash recovery is handled by sentinelOutOfSync → ServerCheckJob → CheckAndStartSentinelJob.
     }
-
-    /**
-     * Determine if a cron schedule should run now.
-     *
-     * When a dedupKey is provided, uses getPreviousRunDate() + last-dispatch tracking
-     * instead of isDue(). This is resilient to queue delays — even if the job is delayed
-     * by minutes, it still catches the missed cron window.
-     */
-    private function shouldRunNow(string $frequency, ?string $timezone = null, ?string $dedupKey = null): bool
-    {
-        $cron = new CronExpression($frequency);
-
-        // Use the frozen execution time, not the current time
-        $baseTime = $this->executionTime ?? Carbon::now();
-        $executionTime = $baseTime->copy()->setTimezone($timezone ?? config('app.timezone'));
-
-        if ($dedupKey === null) {
-            return $cron->isDue($executionTime);
-        }
-
-        $previousDue = Carbon::instance($cron->getPreviousRunDate($executionTime, allowCurrentDate: true));
-
-        $lastDispatched = Cache::get($dedupKey);
-
-        if ($lastDispatched === null) {
-            $isDue = $cron->isDue($executionTime);
-            if ($isDue) {
-                Cache::put($dedupKey, $executionTime->toIso8601String(), 86400);
-            }
-
-            return $isDue;
-        }
-
-        if ($previousDue->gt(Carbon::parse($lastDispatched))) {
-            Cache::put($dedupKey, $executionTime->toIso8601String(), 86400);
-
-            return true;
-        }
-
-        return false;
-    }
 }