feat(jobs): improve scheduled tasks with retry logic and queue cleanup

- Add retry configuration to CoolifyTask (3 tries, 600s timeout)
- Add retry configuration to ScheduledTaskJob (3 tries, configurable timeout)
- Add retry configuration to DatabaseBackupJob (2 tries)
- Implement exponential backoff for all jobs (30s, 60s, 120s intervals)
- Add failed() handlers with comprehensive error logging to scheduled-errors channel
- Add execution tracking: started_at, retry_count, duration (decimal), error_details
- Add configurable timeout field to scheduled tasks (60-3600s, default 300s)
- Update UI to include timeout configuration in task creation/editing forms
- Increase ScheduledJobManager lock expiration from 60s to 90s for high-load environments
- Implement safe queue cleanup with restart vs runtime modes
  - Restart mode: aggressive cleanup (marks all processing jobs as failed)
  - Runtime mode: conservative cleanup (only marks jobs >12h as failed, skips deployments)
- Add cleanup:redis --restart flag for system startup
- Integrate cleanup into Dev.php init() for development environment
- Increase scheduled-errors log retention from 7 to 14 days
- Create comprehensive test suite (unit and feature tests)
- Add TESTING_GUIDE.md with manual testing instructions

Fixes issues with jobs failing after single attempt and "attempted too many times" errors

app/Jobs/CoolifyTask.php

@@ -3,18 +3,35 @@
 namespace App\Jobs;
 
 use App\Actions\CoolifyTask\RunRemoteProcess;
+use App\Enums\ProcessStatus;
 use Illuminate\Bus\Queueable;
 use Illuminate\Contracts\Queue\ShouldBeEncrypted;
 use Illuminate\Contracts\Queue\ShouldQueue;
 use Illuminate\Foundation\Bus\Dispatchable;
 use Illuminate\Queue\InteractsWithQueue;
 use Illuminate\Queue\SerializesModels;
+use Illuminate\Support\Facades\Log;
 use Spatie\Activitylog\Models\Activity;
 
 class CoolifyTask implements ShouldBeEncrypted, ShouldQueue
 {
     use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
 
+    /**
+     * The number of times the job may be attempted.
+     */
+    public $tries = 3;
+
+    /**
+     * The maximum number of unhandled exceptions to allow before failing.
+     */
+    public $maxExceptions = 1;
+
+    /**
+     * The number of seconds the job can run before timing out.
+     */
+    public $timeout = 600;
+
     /**
      * Create a new job instance.
      */
@@ -42,4 +59,36 @@ class CoolifyTask implements ShouldBeEncrypted, ShouldQueue
 
         $remote_process();
     }
+
+    /**
+     * Calculate the number of seconds to wait before retrying the job.
+     */
+    public function backoff(): array
+    {
+        return [30, 90, 180]; // 30s, 90s, 180s between retries
+    }
+
+    /**
+     * Handle a job failure.
+     */
+    public function failed(?\Throwable $exception): void
+    {
+        Log::channel('scheduled-errors')->error('CoolifyTask permanently failed', [
+            'job' => 'CoolifyTask',
+            'activity_id' => $this->activity->id,
+            'server_uuid' => $this->activity->getExtraProperty('server_uuid'),
+            'command_preview' => substr($this->activity->getExtraProperty('command') ?? '', 0, 200),
+            'error' => $exception?->getMessage(),
+            'total_attempts' => $this->attempts(),
+            'trace' => $exception?->getTraceAsString(),
+        ]);
+
+        // Update activity status to reflect permanent failure
+        $this->activity->properties = $this->activity->properties->merge([
+            'status' => ProcessStatus::ERROR->value,
+            'error' => $exception?->getMessage() ?? 'Job permanently failed',
+            'failed_at' => now()->toIso8601String(),
+        ]);
+        $this->activity->save();
+    }
 }

app/Jobs/DatabaseBackupJob.php

@@ -23,6 +23,7 @@ use Illuminate\Contracts\Queue\ShouldQueue;
 use Illuminate\Foundation\Bus\Dispatchable;
 use Illuminate\Queue\InteractsWithQueue;
 use Illuminate\Queue\SerializesModels;
+use Illuminate\Support\Facades\Log;
 use Illuminate\Support\Str;
 use Throwable;
 use Visus\Cuid2\Cuid2;
@@ -31,6 +32,16 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue
 {
     use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
 
+    /**
+     * The number of times the job may be attempted.
+     */
+    public $tries = 2;
+
+    /**
+     * The maximum number of unhandled exceptions to allow before failing.
+     */
+    public $maxExceptions = 1;
+
     public ?Team $team = null;
 
     public Server $server;
@@ -659,17 +670,42 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue
         return "{$helperImage}:{$latestVersion}";
     }
 
+    /**
+     * Calculate the number of seconds to wait before retrying the job.
+     */
+    public function backoff(): array
+    {
+        return [60, 300]; // 1min, 5min between retries
+    }
+
     public function failed(?Throwable $exception): void
     {
+        Log::channel('scheduled-errors')->error('DatabaseBackup permanently failed', [
+            'job' => 'DatabaseBackupJob',
+            'backup_id' => $this->backup->uuid,
+            'database' => $this->database?->name ?? 'unknown',
+            'database_type' => get_class($this->database ?? new \stdClass),
+            'server' => $this->server?->name ?? 'unknown',
+            'total_attempts' => $this->attempts(),
+            'error' => $exception?->getMessage(),
+            'trace' => $exception?->getTraceAsString(),
+        ]);
+
         $log = ScheduledDatabaseBackupExecution::where('uuid', $this->backup_log_uuid)->first();
 
         if ($log) {
             $log->update([
                 'status' => 'failed',
-                'message' => 'Job failed: '.($exception?->getMessage() ?? 'Unknown error'),
+                'message' => 'Job permanently failed after '.$this->attempts().' attempts: '.($exception?->getMessage() ?? 'Unknown error'),
                 'size' => 0,
                 'filename' => null,
+                'finished_at' => Carbon::now(),
             ]);
         }
+
+        // Notify team about permanent failure
+        if ($this->team) {
+            $this->team->notify(new BackupFailed($this->backup, $this->database, $this->backup_output));
+        }
     }
 }

app/Jobs/ScheduledJobManager.php

@@ -52,7 +52,7 @@ class ScheduledJobManager implements ShouldQueue
     {
         return [
             (new WithoutOverlapping('scheduled-job-manager'))
-                ->expireAfter(60)   // Lock expires after 1 minute to prevent stale locks
+                ->expireAfter(90)   // Lock expires after 90s to handle high-load environments with many tasks
                 ->dontRelease(),    // Don't re-queue on lock conflict
         ];
     }

app/Jobs/ScheduledTaskJob.php

@@ -18,11 +18,27 @@ use Illuminate\Contracts\Queue\ShouldQueue;
 use Illuminate\Foundation\Bus\Dispatchable;
 use Illuminate\Queue\InteractsWithQueue;
 use Illuminate\Queue\SerializesModels;
+use Illuminate\Support\Facades\Log;
 
 class ScheduledTaskJob implements ShouldQueue
 {
     use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
 
+    /**
+     * The number of times the job may be attempted.
+     */
+    public $tries = 3;
+
+    /**
+     * The maximum number of unhandled exceptions to allow before failing.
+     */
+    public $maxExceptions = 1;
+
+    /**
+     * The number of seconds the job can run before timing out.
+     */
+    public $timeout = 300;
+
     public Team $team;
 
     public Server $server;
@@ -55,6 +71,9 @@ class ScheduledTaskJob implements ShouldQueue
         }
         $this->team = Team::findOrFail($task->team_id);
         $this->server_timezone = $this->getServerTimezone();
+
+        // Set timeout from task configuration
+        $this->timeout = $this->task->timeout ?? 300;
     }
 
     private function getServerTimezone(): string
@@ -70,9 +89,13 @@ class ScheduledTaskJob implements ShouldQueue
 
     public function handle(): void
     {
+        $startTime = Carbon::now();
+
         try {
             $this->task_log = ScheduledTaskExecution::create([
                 'scheduled_task_id' => $this->task->id,
+                'started_at' => $startTime,
+                'retry_count' => $this->attempts() - 1,
             ]);
 
             $this->server = $this->resource->destination->server;
@@ -129,15 +152,70 @@ class ScheduledTaskJob implements ShouldQueue
                     'message' => $this->task_output ?? $e->getMessage(),
                 ]);
             }
-            $this->team?->notify(new TaskFailed($this->task, $e->getMessage()));
+
+            // Log the error to the scheduled-errors channel
+            Log::channel('scheduled-errors')->error('ScheduledTask execution failed', [
+                'job' => 'ScheduledTaskJob',
+                'task_id' => $this->task->uuid,
+                'task_name' => $this->task->name,
+                'server' => $this->server->name ?? 'unknown',
+                'attempt' => $this->attempts(),
+                'error' => $e->getMessage(),
+            ]);
+
+            // Only notify and throw on final failure
+            if ($this->attempts() >= $this->tries) {
+                $this->team?->notify(new TaskFailed($this->task, $e->getMessage()));
+            }
+
+            // Re-throw to trigger Laravel's retry mechanism with backoff
             throw $e;
         } finally {
             ScheduledTaskDone::dispatch($this->team->id);
             if ($this->task_log) {
+                $finishedAt = Carbon::now();
+                $duration = round($startTime->floatDiffInSeconds($finishedAt), 2);
+
                 $this->task_log->update([
-                    'finished_at' => Carbon::now()->toImmutable(),
+                    'finished_at' => $finishedAt->toImmutable(),
+                    'duration' => $duration,
                 ]);
             }
         }
     }
+
+    /**
+     * Calculate the number of seconds to wait before retrying the job.
+     */
+    public function backoff(): array
+    {
+        return [30, 60, 120]; // 30s, 60s, 120s between retries
+    }
+
+    /**
+     * Handle a job failure.
+     */
+    public function failed(?\Throwable $exception): void
+    {
+        Log::channel('scheduled-errors')->error('ScheduledTask permanently failed', [
+            'job' => 'ScheduledTaskJob',
+            'task_id' => $this->task->uuid,
+            'task_name' => $this->task->name,
+            'server' => $this->server->name ?? 'unknown',
+            'total_attempts' => $this->attempts(),
+            'error' => $exception?->getMessage(),
+            'trace' => $exception?->getTraceAsString(),
+        ]);
+
+        if ($this->task_log) {
+            $this->task_log->update([
+                'status' => 'failed',
+                'message' => 'Job permanently failed after '.$this->attempts().' attempts: '.($exception?->getMessage() ?? 'Unknown error'),
+                'finished_at' => Carbon::now()->toImmutable(),
+            ]);
+        }
+
+        // Notify team about permanent failure
+        $this->team?->notify(new TaskFailed($this->task, $exception?->getMessage() ?? 'Unknown error'));
+    }
 }