mirror of
https://github.com/tiennm99/coolify.git
synced 2026-10-16 00:14:48 +00:00
chore: prepare for PR
This commit is contained in:
1 parent
3eb9426b95
commit
b88f9fca67
6 files changed
+161
-2
No files matched your search
@@ -40,7 +40,7 @@ class Kernel extends ConsoleKernel
|
||||
}
|
||||
|
||||
// $this->scheduleInstance->job(new CleanupStaleMultiplexedConnections)->hourly();
|
||||
$this->scheduleInstance->command('cleanup:redis')->weekly();
|
||||
$this->scheduleInstance->command('cleanup:redis --clear-locks')->daily();
|
||||
|
||||
if (isDev()) {
|
||||
// Instance Jobs
|
||||
|
||||
@@ -15,7 +15,9 @@ use Illuminate\Queue\Middleware\WithoutOverlapping;
|
||||
use Illuminate\Queue\SerializesModels;
|
||||
use Illuminate\Support\Carbon;
|
||||
use Illuminate\Support\Collection;
|
||||
use Illuminate\Support\Facades\Cache;
|
||||
use Illuminate\Support\Facades\Log;
|
||||
use Illuminate\Support\Facades\Redis;
|
||||
|
||||
class ScheduledJobManager implements ShouldQueue
|
||||
{
|
||||
@@ -54,6 +56,11 @@ class ScheduledJobManager implements ShouldQueue
|
||||
*/
|
||||
public function middleware(): array
|
||||
{
|
||||
// Self-healing: clear any stale lock before WithoutOverlapping tries to acquire it.
|
||||
// Stale locks (TTL = -1) can occur during upgrades, Redis restarts, or edge cases.
|
||||
// @see https://github.com/coollabsio/coolify/issues/8327
|
||||
self::clearStaleLockIfPresent();
|
||||
|
||||
return [
|
||||
(new WithoutOverlapping('scheduled-job-manager'))
|
||||
->expireAfter(90) // Lock expires after 90s to handle high-load environments with many tasks
|
||||
@@ -61,6 +68,34 @@ class ScheduledJobManager implements ShouldQueue
|
||||
];
|
||||
}
|
||||
|
||||
/**
|
||||
* Clear a stale WithoutOverlapping lock if it has no TTL (TTL = -1).
|
||||
*
|
||||
* This provides continuous self-healing since it runs every time the job is dispatched.
|
||||
* Stale locks permanently block all scheduled job executions with no user-visible error.
|
||||
*/
|
||||
private static function clearStaleLockIfPresent(): void
|
||||
{
|
||||
try {
|
||||
$cachePrefix = config('cache.prefix', '');
|
||||
$lockKey = $cachePrefix.'laravel-queue-overlap:'.self::class.':scheduled-job-manager';
|
||||
|
||||
$ttl = Redis::connection('default')->ttl($lockKey);
|
||||
|
||||
if ($ttl === -1) {
|
||||
Redis::connection('default')->del($lockKey);
|
||||
Log::channel('scheduled')->warning('Cleared stale ScheduledJobManager lock', [
|
||||
'lock_key' => $lockKey,
|
||||
]);
|
||||
}
|
||||
} catch (\Throwable $e) {
|
||||
// Never let lock cleanup failure prevent the job from running
|
||||
Log::channel('scheduled-errors')->error('Failed to check/clear stale lock', [
|
||||
'error' => $e->getMessage(),
|
||||
]);
|
||||
}
|
||||
}
|
||||
|
||||
public function handle(): void
|
||||
{
|
||||
// Freeze the execution time at the start of the job
|
||||
@@ -108,6 +143,13 @@ class ScheduledJobManager implements ShouldQueue
|
||||
'dispatched' => $this->dispatchedCount,
|
||||
'skipped' => $this->skippedCount,
|
||||
]);
|
||||
|
||||
// Write heartbeat so the UI can detect when the scheduler has stopped
|
||||
try {
|
||||
Cache::put('scheduled-job-manager:heartbeat', now()->toIso8601String(), 300);
|
||||
} catch (\Throwable) {
|
||||
// Non-critical; don't let heartbeat failure affect the job
|
||||
}
|
||||
}
|
||||
|
||||
private function processScheduledBackups(): void
|
||||
|
||||
@@ -3,8 +3,13 @@
|
||||
namespace App\Livewire\Server;
|
||||
|
||||
use App\Jobs\DockerCleanupJob;
|
||||
use App\Models\DockerCleanupExecution;
|
||||
use App\Models\Server;
|
||||
use Cron\CronExpression;
|
||||
use Illuminate\Foundation\Auth\Access\AuthorizesRequests;
|
||||
use Illuminate\Support\Carbon;
|
||||
use Illuminate\Support\Facades\Cache;
|
||||
use Livewire\Attributes\Computed;
|
||||
use Livewire\Attributes\Validate;
|
||||
use Livewire\Component;
|
||||
|
||||
@@ -34,6 +39,53 @@ class DockerCleanup extends Component
|
||||
#[Validate('boolean')]
|
||||
public bool $disableApplicationImageRetention = false;
|
||||
|
||||
#[Computed]
|
||||
public function isCleanupStale(): bool
|
||||
{
|
||||
try {
|
||||
$lastExecution = DockerCleanupExecution::where('server_id', $this->server->id)
|
||||
->orderBy('created_at', 'desc')
|
||||
->first();
|
||||
|
||||
if (! $lastExecution) {
|
||||
return false;
|
||||
}
|
||||
|
||||
$frequency = $this->server->settings->docker_cleanup_frequency ?? '0 0 * * *';
|
||||
if (isset(VALID_CRON_STRINGS[$frequency])) {
|
||||
$frequency = VALID_CRON_STRINGS[$frequency];
|
||||
}
|
||||
|
||||
$cron = new CronExpression($frequency);
|
||||
$now = Carbon::now();
|
||||
$nextRun = Carbon::parse($cron->getNextRunDate($now));
|
||||
$afterThat = Carbon::parse($cron->getNextRunDate($nextRun));
|
||||
$intervalMinutes = $nextRun->diffInMinutes($afterThat);
|
||||
|
||||
$threshold = max($intervalMinutes * 2, 10);
|
||||
|
||||
return Carbon::parse($lastExecution->created_at)->diffInMinutes($now) > $threshold;
|
||||
} catch (\Throwable) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
#[Computed]
|
||||
public function lastExecutionTime(): ?string
|
||||
{
|
||||
return DockerCleanupExecution::where('server_id', $this->server->id)
|
||||
->orderBy('created_at', 'desc')
|
||||
->first()
|
||||
?->created_at
|
||||
?->diffForHumans();
|
||||
}
|
||||
|
||||
#[Computed]
|
||||
public function isSchedulerHealthy(): bool
|
||||
{
|
||||
return Cache::get('scheduled-job-manager:heartbeat') !== null;
|
||||
}
|
||||
|
||||
public function mount(string $server_uuid)
|
||||
{
|
||||
try {
|
||||
|
||||
Reference in new issue
Block a user