mirror of
https://github.com/tiennm99/coolify.git
synced 2026-10-14 08:18:08 +00:00
feat(jobs): add queue delay resilience to scheduled job execution
Implement dedup key-based cron tracking to make scheduled jobs resilient to queue delays. Even if a job is delayed by minutes, it will catch the missed cron window by tracking previousRunDate in cache instead of relying on isDue() alone. - Add dedupKey parameter to shouldRunNow() in ScheduledJobManager - When provided, uses getPreviousRunDate() + cache tracking for resilience - Falls back to isDue() for docker cleanups without dedup key - Prevents double-dispatch within same cron window - Optimize ServerConnectionCheckJob dispatch - Skip SSH checks if Sentinel is healthy (enabled and live) - Reduces redundant checks when Sentinel heartbeat proves connectivity - Remove hourly Sentinel update checks - Consolidate to daily CheckAndStartSentinelJob dispatch - Crash recovery handled by sentinelOutOfSync → ServerCheckJob flow - Add logging for skipped database backups with context (backup_id, database_id, status) - Refactor skip reason methods to accept server parameter, avoiding redundant queries - Add comprehensive test suite for scheduling with various delay scenarios and timezones
This commit is contained in:
1 parent
f68793ed69
commit
a0c177f6f2
5 files changed
+322
-106
No files matched your search
@@ -160,7 +160,8 @@ class ScheduledJobManager implements ShouldQueue
|
||||
|
||||
foreach ($backups as $backup) {
|
||||
try {
|
||||
$skipReason = $this->getBackupSkipReason($backup);
|
||||
$server = $backup->server();
|
||||
$skipReason = $this->getBackupSkipReason($backup, $server);
|
||||
if ($skipReason !== null) {
|
||||
$this->skippedCount++;
|
||||
$this->logSkip('backup', $skipReason, [
|
||||
@@ -173,7 +174,6 @@ class ScheduledJobManager implements ShouldQueue
|
||||
continue;
|
||||
}
|
||||
|
||||
$server = $backup->server();
|
||||
$serverTimezone = data_get($server->settings, 'server_timezone', config('app.timezone'));
|
||||
|
||||
if (validate_timezone($serverTimezone) === false) {
|
||||
@@ -185,7 +185,7 @@ class ScheduledJobManager implements ShouldQueue
|
||||
$frequency = VALID_CRON_STRINGS[$frequency];
|
||||
}
|
||||
|
||||
if ($this->shouldRunNow($frequency, $serverTimezone)) {
|
||||
if ($this->shouldRunNow($frequency, $serverTimezone, "scheduled-backup:{$backup->id}")) {
|
||||
DatabaseBackupJob::dispatch($backup);
|
||||
$this->dispatchedCount++;
|
||||
Log::channel('scheduled')->info('Backup dispatched', [
|
||||
@@ -213,19 +213,19 @@ class ScheduledJobManager implements ShouldQueue
|
||||
|
||||
foreach ($tasks as $task) {
|
||||
try {
|
||||
$skipReason = $this->getTaskSkipReason($task);
|
||||
$server = $task->server();
|
||||
$skipReason = $this->getTaskSkipReason($task, $server);
|
||||
if ($skipReason !== null) {
|
||||
$this->skippedCount++;
|
||||
$this->logSkip('task', $skipReason, [
|
||||
'task_id' => $task->id,
|
||||
'task_name' => $task->name,
|
||||
'team_id' => $task->server()?->team_id,
|
||||
'team_id' => $server?->team_id,
|
||||
]);
|
||||
|
||||
continue;
|
||||
}
|
||||
|
||||
$server = $task->server();
|
||||
$serverTimezone = data_get($server->settings, 'server_timezone', config('app.timezone'));
|
||||
|
||||
if (validate_timezone($serverTimezone) === false) {
|
||||
@@ -237,7 +237,7 @@ class ScheduledJobManager implements ShouldQueue
|
||||
$frequency = VALID_CRON_STRINGS[$frequency];
|
||||
}
|
||||
|
||||
if ($this->shouldRunNow($frequency, $serverTimezone)) {
|
||||
if ($this->shouldRunNow($frequency, $serverTimezone, "scheduled-task:{$task->id}")) {
|
||||
ScheduledTaskJob::dispatch($task);
|
||||
$this->dispatchedCount++;
|
||||
Log::channel('scheduled')->info('Task dispatched', [
|
||||
@@ -256,7 +256,7 @@ class ScheduledJobManager implements ShouldQueue
|
||||
}
|
||||
}
|
||||
|
||||
private function getBackupSkipReason(ScheduledDatabaseBackup $backup): ?string
|
||||
private function getBackupSkipReason(ScheduledDatabaseBackup $backup, ?Server $server): ?string
|
||||
{
|
||||
if (blank(data_get($backup, 'database'))) {
|
||||
$backup->delete();
|
||||
@@ -264,7 +264,6 @@ class ScheduledJobManager implements ShouldQueue
|
||||
return 'database_deleted';
|
||||
}
|
||||
|
||||
$server = $backup->server();
|
||||
if (blank($server)) {
|
||||
$backup->delete();
|
||||
|
||||
@@ -282,12 +281,11 @@ class ScheduledJobManager implements ShouldQueue
|
||||
return null;
|
||||
}
|
||||
|
||||
private function getTaskSkipReason(ScheduledTask $task): ?string
|
||||
private function getTaskSkipReason(ScheduledTask $task, ?Server $server): ?string
|
||||
{
|
||||
$service = $task->service;
|
||||
$application = $task->application;
|
||||
|
||||
$server = $task->server();
|
||||
if (blank($server)) {
|
||||
$task->delete();
|
||||
|
||||
@@ -319,16 +317,38 @@ class ScheduledJobManager implements ShouldQueue
|
||||
return null;
|
||||
}
|
||||
|
||||
private function shouldRunNow(string $frequency, string $timezone): bool
|
||||
/**
|
||||
* Determine if a cron schedule should run now.
|
||||
*
|
||||
* When a dedupKey is provided, uses getPreviousRunDate() + last-dispatch tracking
|
||||
* instead of isDue(). This is resilient to queue delays — even if the job is delayed
|
||||
* by minutes, it still catches the missed cron window. Without dedupKey, falls back
|
||||
* to simple isDue() check.
|
||||
*/
|
||||
private function shouldRunNow(string $frequency, string $timezone, ?string $dedupKey = null): bool
|
||||
{
|
||||
$cron = new CronExpression($frequency);
|
||||
|
||||
// Use the frozen execution time, not the current time
|
||||
// Fallback to current time if execution time is not set (shouldn't happen)
|
||||
$baseTime = $this->executionTime ?? Carbon::now();
|
||||
$executionTime = $baseTime->copy()->setTimezone($timezone);
|
||||
|
||||
return $cron->isDue($executionTime);
|
||||
// No dedup key → simple isDue check (used by docker cleanups)
|
||||
if ($dedupKey === null) {
|
||||
return $cron->isDue($executionTime);
|
||||
}
|
||||
|
||||
// Get the most recent time this cron was due (including current minute)
|
||||
$previousDue = Carbon::instance($cron->getPreviousRunDate($executionTime, allowCurrentDate: true));
|
||||
|
||||
$lastDispatched = Cache::get($dedupKey);
|
||||
|
||||
// Run if: never dispatched before, OR there's been a due time since last dispatch
|
||||
if ($lastDispatched === null || $previousDue->gt(Carbon::parse($lastDispatched))) {
|
||||
Cache::put($dedupKey, $executionTime->toIso8601String(), 86400);
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
private function processDockerCleanups(): void
|
||||
|
||||
Reference in new issue
Block a user