Merge remote-tracking branch 'origin/next' into pr-7764-shadow/add-shared-server-env

app/Jobs/ApplicationPullRequestUpdateJob.php

@@ -47,7 +47,7 @@ class ApplicationPullRequestUpdateJob implements ShouldBeEncrypted, ShouldQueue
             match ($this->status) {
                 ProcessStatus::QUEUED => $this->body = "The preview deployment for **{$serviceName}** is queued. ⏳\n\n",
                 ProcessStatus::IN_PROGRESS => $this->body = "The preview deployment for **{$serviceName}** is in progress. 🟡\n\n",
-                ProcessStatus::FINISHED => $this->body = "The preview deployment for **{$serviceName}** is ready. 🟢\n\n".($this->preview->fqdn ? "[Open Preview]({$this->preview->fqdn}) | " : ''),
+                ProcessStatus::FINISHED => $this->body = "The preview deployment for **{$serviceName}** is ready. 🟢\n\n".$this->getPreviewLinks(),
                 ProcessStatus::ERROR => $this->body = "The preview deployment for **{$serviceName}** failed. 🔴\n\n",
                 ProcessStatus::KILLED => $this->body = "The preview deployment for **{$serviceName}** was killed. ⚫\n\n",
                 ProcessStatus::CANCELLED => $this->body = "The preview deployment for **{$serviceName}** was cancelled. 🚫\n\n",
@@ -91,4 +91,27 @@ class ApplicationPullRequestUpdateJob implements ShouldBeEncrypted, ShouldQueue
     {
         githubApi(source: $this->application->source, endpoint: "/repos/{$this->application->git_repository}/issues/comments/{$this->preview->pull_request_issue_comment_id}", method: 'delete');
     }
+
+    private function getPreviewLinks(): string
+    {
+        if ($this->application->build_pack === 'dockercompose') {
+            $dockerComposeDomains = json_decode($this->preview->docker_compose_domains, true) ?? [];
+            $links = [];
+
+            foreach ($dockerComposeDomains as $serviceName => $config) {
+                $domain = data_get($config, 'domain');
+                if (! empty($domain)) {
+                    $firstDomain = str($domain)->explode(',')->first();
+                    $firstDomain = trim($firstDomain);
+                    if (! empty($firstDomain)) {
+                        $links[] = "[Open {$serviceName}]({$firstDomain})";
+                    }
+                }
+            }
+
+            return ! empty($links) ? implode(' | ', $links).' | ' : '';
+        }
+
+        return $this->preview->fqdn ? "[Open Preview]({$this->preview->fqdn}) | " : '';
+    }
 }

app/Jobs/CheckTraefikVersionForServerJob.php

@@ -6,12 +6,13 @@ use App\Events\ProxyStatusChangedUI;
 use App\Models\Server;
 use App\Notifications\Server\TraefikVersionOutdated;
 use Illuminate\Bus\Queueable;
+use Illuminate\Contracts\Queue\ShouldBeEncrypted;
 use Illuminate\Contracts\Queue\ShouldQueue;
 use Illuminate\Foundation\Bus\Dispatchable;
 use Illuminate\Queue\InteractsWithQueue;
 use Illuminate\Queue\SerializesModels;
 
-class CheckTraefikVersionForServerJob implements ShouldQueue
+class CheckTraefikVersionForServerJob implements ShouldBeEncrypted, ShouldQueue
 {
     use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
 

app/Jobs/CheckTraefikVersionJob.php

@@ -5,12 +5,13 @@ namespace App\Jobs;
 use App\Enums\ProxyTypes;
 use App\Models\Server;
 use Illuminate\Bus\Queueable;
+use Illuminate\Contracts\Queue\ShouldBeEncrypted;
 use Illuminate\Contracts\Queue\ShouldQueue;
 use Illuminate\Foundation\Bus\Dispatchable;
 use Illuminate\Queue\InteractsWithQueue;
 use Illuminate\Queue\SerializesModels;
 
-class CheckTraefikVersionJob implements ShouldQueue
+class CheckTraefikVersionJob implements ShouldBeEncrypted, ShouldQueue
 {
     use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
 

app/Jobs/DatabaseBackupJob.php

@@ -91,7 +91,7 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue
 
                 return;
             }
-            if (data_get($this->backup, 'database_type') === \App\Models\ServiceDatabase::class) {
+            if (data_get($this->backup, 'database_type') === ServiceDatabase::class) {
                 $this->database = data_get($this->backup, 'database');
                 $this->server = $this->database->service->server;
                 $this->s3 = $this->backup->s3;
@@ -111,9 +111,15 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue
 
             $status = str(data_get($this->database, 'status'));
             if (! $status->startsWith('running') && $this->database->id !== 0) {
+                Log::info('DatabaseBackupJob skipped: database not running', [
+                    'backup_id' => $this->backup->id,
+                    'database_id' => $this->database->id,
+                    'status' => (string) $status,
+                ]);
+
                 return;
             }
-            if (data_get($this->backup, 'database_type') === \App\Models\ServiceDatabase::class) {
+            if (data_get($this->backup, 'database_type') === ServiceDatabase::class) {
                 $databaseType = $this->database->databaseType();
                 $serviceUuid = $this->database->service->uuid;
                 $serviceName = str($this->database->service->name)->slug();
@@ -235,7 +241,7 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue
                             }
                         }
 
-                    } catch (\Throwable $e) {
+                    } catch (Throwable $e) {
                         // Continue without env vars - will be handled in backup_standalone_mongodb method
                     }
                 }
@@ -382,7 +388,7 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue
                     } else {
                         throw new \Exception('Local backup file is empty or was not created');
                     }
-                } catch (\Throwable $e) {
+                } catch (Throwable $e) {
                     // Local backup failed
                     if ($this->backup_log) {
                         $this->backup_log->update([
@@ -393,7 +399,15 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue
                             's3_uploaded' => null,
                         ]);
                     }
-                    $this->team?->notify(new BackupFailed($this->backup, $this->database, $this->error_output ?? $this->backup_output ?? $e->getMessage(), $database));
+                    try {
+                        $this->team?->notify(new BackupFailed($this->backup, $this->database, $this->error_output ?? $this->backup_output ?? $e->getMessage(), $database));
+                    } catch (Throwable $notifyException) {
+                        Log::channel('scheduled-errors')->warning('Failed to send backup failure notification', [
+                            'backup_id' => $this->backup->uuid,
+                            'database' => $database,
+                            'error' => $notifyException->getMessage(),
+                        ]);
+                    }
 
                     continue;
                 }
@@ -409,7 +423,7 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue
                             deleteBackupsLocally($this->backup_location, $this->server);
                             $localStorageDeleted = true;
                         }
-                    } catch (\Throwable $e) {
+                    } catch (Throwable $e) {
                         // S3 upload failed but local backup succeeded
                         $s3UploadError = $e->getMessage();
                     }
@@ -433,18 +447,27 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue
                         'local_storage_deleted' => $localStorageDeleted,
                     ]);
 
-                    // Send appropriate notification
-                    if ($s3UploadError) {
-                        $this->team->notify(new BackupSuccessWithS3Warning($this->backup, $this->database, $database, $s3UploadError));
-                    } else {
-                        $this->team->notify(new BackupSuccess($this->backup, $this->database, $database));
+                    // Send appropriate notification (wrapped in try-catch so notification
+                    // failures never affect backup status — see GitHub issue #9088)
+                    try {
+                        if ($s3UploadError) {
+                            $this->team->notify(new BackupSuccessWithS3Warning($this->backup, $this->database, $database, $s3UploadError));
+                        } else {
+                            $this->team->notify(new BackupSuccess($this->backup, $this->database, $database));
+                        }
+                    } catch (Throwable $e) {
+                        Log::channel('scheduled-errors')->warning('Failed to send backup success notification', [
+                            'backup_id' => $this->backup->uuid,
+                            'database' => $database,
+                            'error' => $e->getMessage(),
+                        ]);
                     }
                 }
             }
             if ($this->backup_log && $this->backup_log->status === 'success') {
                 removeOldBackups($this->backup);
             }
-        } catch (\Throwable $e) {
+        } catch (Throwable $e) {
             throw $e;
         } finally {
             if ($this->team) {
@@ -466,19 +489,23 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue
                 // For service-based MongoDB, try to build URL from environment variables
                 if (filled($this->mongo_root_username) && filled($this->mongo_root_password)) {
                     // Use container name instead of server IP for service-based MongoDB
-                    $url = "mongodb://{$this->mongo_root_username}:{$this->mongo_root_password}@{$this->container_name}:27017";
+                    // URL-encode credentials to prevent URI injection
+                    $encodedUser = rawurlencode($this->mongo_root_username);
+                    $encodedPass = rawurlencode($this->mongo_root_password);
+                    $url = "mongodb://{$encodedUser}:{$encodedPass}@{$this->container_name}:27017";
                 } else {
                     // If no environment variables are available, throw an exception
                     throw new \Exception('MongoDB credentials not found. Ensure MONGO_INITDB_ROOT_USERNAME and MONGO_INITDB_ROOT_PASSWORD environment variables are available in the container.');
                 }
             }
-            \Log::info('MongoDB backup URL configured', ['has_url' => filled($url), 'using_env_vars' => blank($this->database->internal_db_url)]);
+            Log::info('MongoDB backup URL configured', ['has_url' => filled($url), 'using_env_vars' => blank($this->database->internal_db_url)]);
+            $escapedUrl = escapeshellarg($url);
             if ($databaseWithCollections === 'all') {
                 $commands[] = 'mkdir -p '.$this->backup_dir;
                 if (str($this->database->image)->startsWith('mongo:4')) {
-                    $commands[] = "docker exec $this->container_name mongodump --uri=\"$url\" --gzip --archive > $this->backup_location";
+                    $commands[] = "docker exec $this->container_name mongodump --uri=$escapedUrl --gzip --archive > $this->backup_location";
                 } else {
-                    $commands[] = "docker exec $this->container_name mongodump --authenticationDatabase=admin --uri=\"$url\" --gzip --archive > $this->backup_location";
+                    $commands[] = "docker exec $this->container_name mongodump --authenticationDatabase=admin --uri=$escapedUrl --gzip --archive > $this->backup_location";
                 }
             } else {
                 if (str($databaseWithCollections)->contains(':')) {
@@ -496,15 +523,23 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue
 
                 if ($collectionsToExclude->count() === 0) {
                     if (str($this->database->image)->startsWith('mongo:4')) {
-                        $commands[] = "docker exec $this->container_name mongodump --uri=\"$url\" --gzip --archive > $this->backup_location";
+                        $commands[] = "docker exec $this->container_name mongodump --uri=$escapedUrl --gzip --archive > $this->backup_location";
                     } else {
-                        $commands[] = "docker exec $this->container_name mongodump --authenticationDatabase=admin --uri=\"$url\" --db $escapedDatabaseName --gzip --archive > $this->backup_location";
+                        $commands[] = "docker exec $this->container_name mongodump --authenticationDatabase=admin --uri=$escapedUrl --db $escapedDatabaseName --gzip --archive > $this->backup_location";
                     }
                 } else {
+                    // Validate and escape each collection name
+                    $escapedCollections = $collectionsToExclude->map(function ($collection) {
+                        $collection = trim($collection);
+                        validateShellSafePath($collection, 'collection name');
+
+                        return escapeshellarg($collection);
+                    });
+
                     if (str($this->database->image)->startsWith('mongo:4')) {
-                        $commands[] = "docker exec $this->container_name mongodump --uri=$url --gzip --excludeCollection ".$collectionsToExclude->implode(' --excludeCollection ')." --archive > $this->backup_location";
+                        $commands[] = "docker exec $this->container_name mongodump --uri=$escapedUrl --gzip --excludeCollection ".$escapedCollections->implode(' --excludeCollection ')." --archive > $this->backup_location";
                     } else {
-                        $commands[] = "docker exec $this->container_name mongodump --authenticationDatabase=admin --uri=\"$url\" --db $escapedDatabaseName --gzip --excludeCollection ".$collectionsToExclude->implode(' --excludeCollection ')." --archive > $this->backup_location";
+                        $commands[] = "docker exec $this->container_name mongodump --authenticationDatabase=admin --uri=$escapedUrl --db $escapedDatabaseName --gzip --excludeCollection ".$escapedCollections->implode(' --excludeCollection ')." --archive > $this->backup_location";
                     }
                 }
             }
@@ -513,7 +548,7 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue
             if ($this->backup_output === '') {
                 $this->backup_output = null;
             }
-        } catch (\Throwable $e) {
+        } catch (Throwable $e) {
             $this->add_to_error_output($e->getMessage());
             throw $e;
         }
@@ -525,15 +560,16 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue
             $commands[] = 'mkdir -p '.$this->backup_dir;
             $backupCommand = 'docker exec';
             if ($this->postgres_password) {
-                $backupCommand .= " -e PGPASSWORD=\"{$this->postgres_password}\"";
+                $backupCommand .= ' -e PGPASSWORD='.escapeshellarg($this->postgres_password);
             }
+            $escapedUsername = escapeshellarg($this->database->postgres_user);
             if ($this->backup->dump_all) {
-                $backupCommand .= " $this->container_name pg_dumpall --username {$this->database->postgres_user} | gzip > $this->backup_location";
+                $backupCommand .= " $this->container_name pg_dumpall --username $escapedUsername | gzip > $this->backup_location";
             } else {
                 // Validate and escape database name to prevent command injection
                 validateShellSafePath($database, 'database name');
                 $escapedDatabase = escapeshellarg($database);
-                $backupCommand .= " $this->container_name pg_dump --format=custom --no-acl --no-owner --username {$this->database->postgres_user} $escapedDatabase > $this->backup_location";
+                $backupCommand .= " $this->container_name pg_dump --format=custom --no-acl --no-owner --username $escapedUsername $escapedDatabase > $this->backup_location";
             }
 
             $commands[] = $backupCommand;
@@ -542,7 +578,7 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue
             if ($this->backup_output === '') {
                 $this->backup_output = null;
             }
-        } catch (\Throwable $e) {
+        } catch (Throwable $e) {
             $this->add_to_error_output($e->getMessage());
             throw $e;
         }
@@ -552,20 +588,21 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue
     {
         try {
             $commands[] = 'mkdir -p '.$this->backup_dir;
+            $escapedPassword = escapeshellarg($this->database->mysql_root_password);
             if ($this->backup->dump_all) {
-                $commands[] = "docker exec $this->container_name mysqldump -u root -p\"{$this->database->mysql_root_password}\" --all-databases --single-transaction --quick --lock-tables=false --compress | gzip > $this->backup_location";
+                $commands[] = "docker exec $this->container_name mysqldump -u root -p$escapedPassword --all-databases --single-transaction --quick --lock-tables=false --compress | gzip > $this->backup_location";
             } else {
                 // Validate and escape database name to prevent command injection
                 validateShellSafePath($database, 'database name');
                 $escapedDatabase = escapeshellarg($database);
-                $commands[] = "docker exec $this->container_name mysqldump -u root -p\"{$this->database->mysql_root_password}\" $escapedDatabase > $this->backup_location";
+                $commands[] = "docker exec $this->container_name mysqldump -u root -p$escapedPassword $escapedDatabase > $this->backup_location";
             }
             $this->backup_output = instant_remote_process($commands, $this->server, true, false, $this->timeout, disableMultiplexing: true);
             $this->backup_output = trim($this->backup_output);
             if ($this->backup_output === '') {
                 $this->backup_output = null;
             }
-        } catch (\Throwable $e) {
+        } catch (Throwable $e) {
             $this->add_to_error_output($e->getMessage());
             throw $e;
         }
@@ -575,20 +612,21 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue
     {
         try {
             $commands[] = 'mkdir -p '.$this->backup_dir;
+            $escapedPassword = escapeshellarg($this->database->mariadb_root_password);
             if ($this->backup->dump_all) {
-                $commands[] = "docker exec $this->container_name mariadb-dump -u root -p\"{$this->database->mariadb_root_password}\" --all-databases --single-transaction --quick --lock-tables=false --compress > $this->backup_location";
+                $commands[] = "docker exec $this->container_name mariadb-dump -u root -p$escapedPassword --all-databases --single-transaction --quick --lock-tables=false --compress > $this->backup_location";
             } else {
                 // Validate and escape database name to prevent command injection
                 validateShellSafePath($database, 'database name');
                 $escapedDatabase = escapeshellarg($database);
-                $commands[] = "docker exec $this->container_name mariadb-dump -u root -p\"{$this->database->mariadb_root_password}\" $escapedDatabase > $this->backup_location";
+                $commands[] = "docker exec $this->container_name mariadb-dump -u root -p$escapedPassword $escapedDatabase > $this->backup_location";
             }
             $this->backup_output = instant_remote_process($commands, $this->server, true, false, $this->timeout, disableMultiplexing: true);
             $this->backup_output = trim($this->backup_output);
             if ($this->backup_output === '') {
                 $this->backup_output = null;
             }
-        } catch (\Throwable $e) {
+        } catch (Throwable $e) {
             $this->add_to_error_output($e->getMessage());
             throw $e;
         }
@@ -619,21 +657,28 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue
 
     private function upload_to_s3(): void
     {
+        if (is_null($this->s3)) {
+            $this->backup->update([
+                'save_s3' => false,
+                's3_storage_id' => null,
+            ]);
+
+            throw new \Exception('S3 storage configuration is missing or has been deleted (S3 storage ID: '.($this->backup->s3_storage_id ?? 'null').'). S3 backup has been disabled for this schedule.');
+        }
+
         try {
-            if (is_null($this->s3)) {
-                return;
-            }
             $key = $this->s3->key;
             $secret = $this->s3->secret;
             // $region = $this->s3->region;
             $bucket = $this->s3->bucket;
             $endpoint = $this->s3->endpoint;
             $this->s3->testConnection(shouldSave: true);
-            if (data_get($this->backup, 'database_type') === \App\Models\ServiceDatabase::class) {
+            if (data_get($this->backup, 'database_type') === ServiceDatabase::class) {
                 $network = $this->database->service->destination->network;
             } else {
                 $network = $this->database->destination->network;
             }
+            $safeNetwork = escapeshellarg($network);
 
             $fullImageName = $this->getFullImageName();
 
@@ -645,13 +690,13 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue
             if (isDev()) {
                 if ($this->database->name === 'coolify-db') {
                     $backup_location_from = '/var/lib/docker/volumes/coolify_dev_backups_data/_data/coolify/coolify-db-'.$this->server->ip.$this->backup_file;
-                    $commands[] = "docker run -d --network {$network} --name backup-of-{$this->backup_log_uuid} --rm -v $backup_location_from:$this->backup_location:ro {$fullImageName}";
+                    $commands[] = "docker run -d --network {$safeNetwork} --name backup-of-{$this->backup_log_uuid} --rm -v $backup_location_from:$this->backup_location:ro {$fullImageName}";
                 } else {
                     $backup_location_from = '/var/lib/docker/volumes/coolify_dev_backups_data/_data/databases/'.str($this->team->name)->slug().'-'.$this->team->id.'/'.$this->directory_name.$this->backup_file;
-                    $commands[] = "docker run -d --network {$network} --name backup-of-{$this->backup_log_uuid} --rm -v $backup_location_from:$this->backup_location:ro {$fullImageName}";
+                    $commands[] = "docker run -d --network {$safeNetwork} --name backup-of-{$this->backup_log_uuid} --rm -v $backup_location_from:$this->backup_location:ro {$fullImageName}";
                 }
             } else {
-                $commands[] = "docker run -d --network {$network} --name backup-of-{$this->backup_log_uuid} --rm -v $this->backup_location:$this->backup_location:ro {$fullImageName}";
+                $commands[] = "docker run -d --network {$safeNetwork} --name backup-of-{$this->backup_log_uuid} --rm -v $this->backup_location:$this->backup_location:ro {$fullImageName}";
             }
 
             // Escape S3 credentials to prevent command injection
@@ -664,7 +709,7 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue
             instant_remote_process($commands, $this->server, true, false, null, disableMultiplexing: true);
 
             $this->s3_uploaded = true;
-        } catch (\Throwable $e) {
+        } catch (Throwable $e) {
             $this->s3_uploaded = false;
             $this->add_to_error_output($e->getMessage());
             throw $e;
@@ -698,20 +743,32 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue
         $log = ScheduledDatabaseBackupExecution::where('uuid', $this->backup_log_uuid)->first();
 
         if ($log) {
-            $log->update([
-                'status' => 'failed',
-                'message' => 'Job permanently failed after '.$this->attempts().' attempts: '.($exception?->getMessage() ?? 'Unknown error'),
-                'size' => 0,
-                'filename' => null,
-                'finished_at' => Carbon::now(),
-            ]);
+            // Don't overwrite a successful backup status — a post-backup error
+            // (e.g. notification failure) should not retroactively mark the backup
+            // as failed (see GitHub issue #9088)
+            if ($log->status !== 'success') {
+                $log->update([
+                    'status' => 'failed',
+                    'message' => 'Job permanently failed after '.$this->attempts().' attempts: '.($exception?->getMessage() ?? 'Unknown error'),
+                    'size' => 0,
+                    'filename' => null,
+                    'finished_at' => Carbon::now(),
+                ]);
+            }
         }
 
-        // Notify team about permanent failure
-        if ($this->team) {
+        // Notify team about permanent failure (only if backup didn't already succeed)
+        if ($this->team && $log?->status !== 'success') {
             $databaseName = $log?->database_name ?? 'unknown';
             $output = $this->backup_output ?? $exception?->getMessage() ?? 'Unknown error';
-            $this->team->notify(new BackupFailed($this->backup, $this->database, $output, $databaseName));
+            try {
+                $this->team->notify(new BackupFailed($this->backup, $this->database, $output, $databaseName));
+            } catch (Throwable $e) {
+                Log::channel('scheduled-errors')->warning('Failed to send backup permanent failure notification', [
+                    'backup_id' => $this->backup->uuid,
+                    'error' => $e->getMessage(),
+                ]);
+            }
         }
     }
 }

app/Jobs/DockerCleanupJob.php

@@ -39,19 +39,27 @@ class DockerCleanupJob implements ShouldBeEncrypted, ShouldQueue
         public bool $manualCleanup = false,
         public bool $deleteUnusedVolumes = false,
         public bool $deleteUnusedNetworks = false
-    ) {}
+    ) {
+        $this->onQueue('high');
+    }
 
     public function handle(): void
     {
         try {
-            if (! $this->server->isFunctional()) {
-                return;
-            }
-
             $this->execution_log = DockerCleanupExecution::create([
                 'server_id' => $this->server->id,
             ]);
 
+            if (! $this->server->isFunctional()) {
+                $this->execution_log->update([
+                    'status' => 'failed',
+                    'message' => 'Server is not functional (unreachable, unusable, or disabled)',
+                    'finished_at' => Carbon::now()->toImmutable(),
+                ]);
+
+                return;
+            }
+
             $this->usageBefore = $this->server->getDiskUsage();
 
             if ($this->manualCleanup || $this->server->settings->force_docker_cleanup) {
@@ -91,6 +99,8 @@ class DockerCleanupJob implements ShouldBeEncrypted, ShouldQueue
 
                 $this->server->team?->notify(new DockerCleanupSuccess($this->server, $message));
                 event(new DockerCleanupDone($this->execution_log));
+
+                return;
             }
 
             if ($this->usageBefore >= $this->server->settings->docker_cleanup_threshold) {

app/Jobs/ProcessGithubPullRequestWebhook.php

@@ -118,7 +118,7 @@ class ProcessGithubPullRequestWebhook implements ShouldBeEncrypted, ShouldQueue
 
         if (! $found) {
             if ($application->build_pack === 'dockercompose') {
-                $preview = ApplicationPreview::create([
+                $preview = ApplicationPreview::forceCreate([
                     'git_type' => 'github',
                     'application_id' => $application->id,
                     'pull_request_id' => $this->pullRequestId,
@@ -127,7 +127,7 @@ class ProcessGithubPullRequestWebhook implements ShouldBeEncrypted, ShouldQueue
                 ]);
                 $preview->generate_preview_fqdn_compose();
             } else {
-                $preview = ApplicationPreview::create([
+                $preview = ApplicationPreview::forceCreate([
                     'git_type' => 'github',
                     'application_id' => $application->id,
                     'pull_request_id' => $this->pullRequestId,

app/Jobs/PushServerUpdateJob.php

@@ -24,6 +24,7 @@ use Illuminate\Queue\InteractsWithQueue;
 use Illuminate\Queue\Middleware\WithoutOverlapping;
 use Illuminate\Queue\SerializesModels;
 use Illuminate\Support\Collection;
+use Illuminate\Support\Facades\Cache;
 use Laravel\Horizon\Contracts\Silenced;
 
 class PushServerUpdateJob implements ShouldBeEncrypted, ShouldQueue, Silenced
@@ -130,7 +131,14 @@ class PushServerUpdateJob implements ShouldBeEncrypted, ShouldQueue, Silenced
 
         $this->containers = collect(data_get($data, 'containers'));
         $filesystemUsageRoot = data_get($data, 'filesystem_usage_root.used_percentage');
-        ServerStorageCheckJob::dispatch($this->server, $filesystemUsageRoot);
+
+        // Only dispatch storage check when disk percentage actually changes
+        $storageCacheKey = 'storage-check:'.$this->server->id;
+        $lastPercentage = Cache::get($storageCacheKey);
+        if ($lastPercentage === null || (string) $lastPercentage !== (string) $filesystemUsageRoot) {
+            Cache::put($storageCacheKey, $filesystemUsageRoot, 600);
+            ServerStorageCheckJob::dispatch($this->server, $filesystemUsageRoot);
+        }
 
         if ($this->containers->isEmpty()) {
             return;
@@ -207,6 +215,9 @@ class PushServerUpdateJob implements ShouldBeEncrypted, ShouldQueue, Silenced
                 $serviceId = $labels->get('coolify.serviceId');
                 $subType = $labels->get('coolify.service.subType');
                 $subId = $labels->get('coolify.service.subId');
+                if (empty(trim((string) $subId))) {
+                    continue;
+                }
                 if ($subType === 'application') {
                     $this->foundServiceApplicationIds->push($subId);
                     // Store container status for aggregation
@@ -296,6 +307,8 @@ class PushServerUpdateJob implements ShouldBeEncrypted, ShouldQueue, Silenced
                 if ($aggregatedStatus && $application->status !== $aggregatedStatus) {
                     $application->status = $aggregatedStatus;
                     $application->save();
+                } elseif ($aggregatedStatus) {
+                    $application->update(['last_online_at' => now()]);
                 }
 
                 continue;
@@ -310,6 +323,8 @@ class PushServerUpdateJob implements ShouldBeEncrypted, ShouldQueue, Silenced
             if ($aggregatedStatus && $application->status !== $aggregatedStatus) {
                 $application->status = $aggregatedStatus;
                 $application->save();
+            } elseif ($aggregatedStatus) {
+                $application->update(['last_online_at' => now()]);
             }
         }
     }
@@ -324,6 +339,10 @@ class PushServerUpdateJob implements ShouldBeEncrypted, ShouldQueue, Silenced
             // Parse key: serviceId:subType:subId
             [$serviceId, $subType, $subId] = explode(':', $key);
 
+            if (empty($subId)) {
+                continue;
+            }
+
             $service = $this->services->where('id', $serviceId)->first();
             if (! $service) {
                 continue;
@@ -332,9 +351,9 @@ class PushServerUpdateJob implements ShouldBeEncrypted, ShouldQueue, Silenced
             // Get the service sub-resource (ServiceApplication or ServiceDatabase)
             $subResource = null;
             if ($subType === 'application') {
-                $subResource = $service->applications()->where('id', $subId)->first();
+                $subResource = $service->applications->where('id', $subId)->first();
             } elseif ($subType === 'database') {
-                $subResource = $service->databases()->where('id', $subId)->first();
+                $subResource = $service->databases->where('id', $subId)->first();
             }
 
             if (! $subResource) {
@@ -356,6 +375,8 @@ class PushServerUpdateJob implements ShouldBeEncrypted, ShouldQueue, Silenced
                 if ($aggregatedStatus && $subResource->status !== $aggregatedStatus) {
                     $subResource->status = $aggregatedStatus;
                     $subResource->save();
+                } elseif ($aggregatedStatus) {
+                    $subResource->update(['last_online_at' => now()]);
                 }
 
                 continue;
@@ -371,6 +392,8 @@ class PushServerUpdateJob implements ShouldBeEncrypted, ShouldQueue, Silenced
             if ($aggregatedStatus && $subResource->status !== $aggregatedStatus) {
                 $subResource->status = $aggregatedStatus;
                 $subResource->save();
+            } elseif ($aggregatedStatus) {
+                $subResource->update(['last_online_at' => now()]);
             }
         }
     }
@@ -384,6 +407,8 @@ class PushServerUpdateJob implements ShouldBeEncrypted, ShouldQueue, Silenced
         if ($application->status !== $containerStatus) {
             $application->status = $containerStatus;
             $application->save();
+        } else {
+            $application->update(['last_online_at' => now()]);
         }
     }
 
@@ -398,6 +423,8 @@ class PushServerUpdateJob implements ShouldBeEncrypted, ShouldQueue, Silenced
         if ($application->status !== $containerStatus) {
             $application->status = $containerStatus;
             $application->save();
+        } else {
+            $application->update(['last_online_at' => now()]);
         }
     }
 
@@ -473,8 +500,13 @@ class PushServerUpdateJob implements ShouldBeEncrypted, ShouldQueue, Silenced
                 } catch (\Throwable $e) {
                 }
             } else {
-                // Connect proxy to networks asynchronously to avoid blocking the status update
-                ConnectProxyToNetworksJob::dispatch($this->server);
+                // Connect proxy to networks periodically (every 10 min) to avoid excessive job dispatches.
+                // On-demand triggers (new network, service deploy) use dispatchSync() and bypass this.
+                $proxyCacheKey = 'connect-proxy:'.$this->server->id;
+                if (! Cache::has($proxyCacheKey)) {
+                    Cache::put($proxyCacheKey, true, 600);
+                    ConnectProxyToNetworksJob::dispatch($this->server);
+                }
             }
         }
     }
@@ -488,6 +520,8 @@ class PushServerUpdateJob implements ShouldBeEncrypted, ShouldQueue, Silenced
         if ($database->status !== $containerStatus) {
             $database->status = $containerStatus;
             $database->save();
+        } else {
+            $database->update(['last_online_at' => now()]);
         }
         if ($this->isRunning($containerStatus) && $tcpProxy) {
             $tcpProxyContainerFound = $this->containers->filter(function ($value, $key) use ($databaseUuid) {
@@ -496,7 +530,14 @@ class PushServerUpdateJob implements ShouldBeEncrypted, ShouldQueue, Silenced
             if (! $tcpProxyContainerFound) {
                 StartDatabaseProxy::dispatch($database);
                 $this->server->team?->notify(new ContainerRestarted("TCP Proxy for {$database->name}", $this->server));
-            } else {
+            }
+        } elseif ($this->isRunning($containerStatus) && ! $tcpProxy) {
+            // Clean up orphaned proxy containers when is_public=false
+            $orphanedProxy = $this->containers->filter(function ($value, $key) use ($databaseUuid) {
+                return data_get($value, 'name') === "$databaseUuid-proxy" && data_get($value, 'state') === 'running';
+            })->first();
+            if ($orphanedProxy) {
+                StopDatabaseProxy::dispatch($database);
             }
         }
     }
@@ -518,8 +559,12 @@ class PushServerUpdateJob implements ShouldBeEncrypted, ShouldQueue, Silenced
             $database = $this->databases->where('uuid', $databaseUuid)->first();
             if ($database) {
                 if (! str($database->status)->startsWith('exited')) {
-                    $database->status = 'exited';
-                    $database->save();
+                    $database->update([
+                        'status' => 'exited',
+                        'restart_count' => 0,
+                        'last_restart_at' => null,
+                        'last_restart_type' => null,
+                    ]);
                 }
                 if ($database->is_public) {
                     StopDatabaseProxy::dispatch($database);
@@ -528,31 +573,6 @@ class PushServerUpdateJob implements ShouldBeEncrypted, ShouldQueue, Silenced
         });
     }
 
-    private function updateServiceSubStatus(string $serviceId, string $subType, string $subId, string $containerStatus)
-    {
-        $service = $this->services->where('id', $serviceId)->first();
-        if (! $service) {
-            return;
-        }
-        if ($subType === 'application') {
-            $application = $service->applications()->where('id', $subId)->first();
-            if ($application) {
-                if ($application->status !== $containerStatus) {
-                    $application->status = $containerStatus;
-                    $application->save();
-                }
-            }
-        } elseif ($subType === 'database') {
-            $database = $service->databases()->where('id', $subId)->first();
-            if ($database) {
-                if ($database->status !== $containerStatus) {
-                    $database->status = $containerStatus;
-                    $database->save();
-                }
-            }
-        }
-    }
-
     private function updateNotFoundServiceStatus()
     {
         $notFoundServiceApplicationIds = $this->allServiceApplicationIds->diff($this->foundServiceApplicationIds);

app/Jobs/RegenerateSslCertJob.php

@@ -7,13 +7,14 @@ use App\Models\SslCertificate;
 use App\Models\Team;
 use App\Notifications\SslExpirationNotification;
 use Illuminate\Bus\Queueable;
+use Illuminate\Contracts\Queue\ShouldBeEncrypted;
 use Illuminate\Contracts\Queue\ShouldQueue;
 use Illuminate\Foundation\Bus\Dispatchable;
 use Illuminate\Queue\InteractsWithQueue;
 use Illuminate\Queue\SerializesModels;
 use Illuminate\Support\Facades\Log;
 
-class RegenerateSslCertJob implements ShouldQueue
+class RegenerateSslCertJob implements ShouldBeEncrypted, ShouldQueue
 {
     use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
 

app/Jobs/ScheduledJobManager.php

@@ -6,7 +6,6 @@ use App\Models\ScheduledDatabaseBackup;
 use App\Models\ScheduledTask;
 use App\Models\Server;
 use App\Models\Team;
-use Cron\CronExpression;
 use Illuminate\Bus\Queueable;
 use Illuminate\Contracts\Queue\ShouldQueue;
 use Illuminate\Foundation\Bus\Dispatchable;
@@ -15,7 +14,9 @@ use Illuminate\Queue\Middleware\WithoutOverlapping;
 use Illuminate\Queue\SerializesModels;
 use Illuminate\Support\Carbon;
 use Illuminate\Support\Collection;
+use Illuminate\Support\Facades\Cache;
 use Illuminate\Support\Facades\Log;
+use Illuminate\Support\Facades\Redis;
 
 class ScheduledJobManager implements ShouldQueue
 {
@@ -27,6 +28,10 @@ class ScheduledJobManager implements ShouldQueue
      */
     private ?Carbon $executionTime = null;
 
+    private int $dispatchedCount = 0;
+
+    private int $skippedCount = 0;
+
     /**
      * Create a new job instance.
      */
@@ -50,6 +55,11 @@ class ScheduledJobManager implements ShouldQueue
      */
     public function middleware(): array
     {
+        // Self-healing: clear any stale lock before WithoutOverlapping tries to acquire it.
+        // Stale locks (TTL = -1) can occur during upgrades, Redis restarts, or edge cases.
+        // @see https://github.com/coollabsio/coolify/issues/8327
+        self::clearStaleLockIfPresent();
+
         return [
             (new WithoutOverlapping('scheduled-job-manager'))
                 ->expireAfter(90)   // Lock expires after 90s to handle high-load environments with many tasks
@@ -57,10 +67,44 @@ class ScheduledJobManager implements ShouldQueue
         ];
     }
 
+    /**
+     * Clear a stale WithoutOverlapping lock if it has no TTL (TTL = -1).
+     *
+     * This provides continuous self-healing since it runs every time the job is dispatched.
+     * Stale locks permanently block all scheduled job executions with no user-visible error.
+     */
+    private static function clearStaleLockIfPresent(): void
+    {
+        try {
+            $cachePrefix = config('cache.prefix', '');
+            $lockKey = $cachePrefix.'laravel-queue-overlap:'.self::class.':scheduled-job-manager';
+
+            $ttl = Redis::connection('default')->ttl($lockKey);
+
+            if ($ttl === -1) {
+                Redis::connection('default')->del($lockKey);
+                Log::channel('scheduled')->warning('Cleared stale ScheduledJobManager lock', [
+                    'lock_key' => $lockKey,
+                ]);
+            }
+        } catch (\Throwable $e) {
+            // Never let lock cleanup failure prevent the job from running
+            Log::channel('scheduled-errors')->error('Failed to check/clear stale lock', [
+                'error' => $e->getMessage(),
+            ]);
+        }
+    }
+
     public function handle(): void
     {
         // Freeze the execution time at the start of the job
         $this->executionTime = Carbon::now();
+        $this->dispatchedCount = 0;
+        $this->skippedCount = 0;
+
+        Log::channel('scheduled')->info('ScheduledJobManager started', [
+            'execution_time' => $this->executionTime->toIso8601String(),
+        ]);
 
         // Process backups - don't let failures stop task processing
         try {
@@ -91,6 +135,20 @@ class ScheduledJobManager implements ShouldQueue
                 'trace' => $e->getTraceAsString(),
             ]);
         }
+
+        Log::channel('scheduled')->info('ScheduledJobManager completed', [
+            'execution_time' => $this->executionTime->toIso8601String(),
+            'duration_ms' => $this->executionTime->diffInMilliseconds(Carbon::now()),
+            'dispatched' => $this->dispatchedCount,
+            'skipped' => $this->skippedCount,
+        ]);
+
+        // Write heartbeat so the UI can detect when the scheduler has stopped
+        try {
+            Cache::put('scheduled-job-manager:heartbeat', now()->toIso8601String(), 300);
+        } catch (\Throwable) {
+            // Non-critical; don't let heartbeat failure affect the job
+        }
     }
 
     private function processScheduledBackups(): void
@@ -101,12 +159,20 @@ class ScheduledJobManager implements ShouldQueue
 
         foreach ($backups as $backup) {
             try {
-                // Apply the same filtering logic as the original
-                if (! $this->shouldProcessBackup($backup)) {
+                $server = $backup->server();
+                $skipReason = $this->getBackupSkipReason($backup, $server);
+                if ($skipReason !== null) {
+                    $this->skippedCount++;
+                    $this->logSkip('backup', $skipReason, [
+                        'backup_id' => $backup->id,
+                        'database_id' => $backup->database_id,
+                        'database_type' => $backup->database_type,
+                        'team_id' => $backup->team_id ?? null,
+                    ]);
+
                     continue;
                 }
 
-                $server = $backup->server();
                 $serverTimezone = data_get($server->settings, 'server_timezone', config('app.timezone'));
 
                 if (validate_timezone($serverTimezone) === false) {
@@ -118,8 +184,16 @@ class ScheduledJobManager implements ShouldQueue
                     $frequency = VALID_CRON_STRINGS[$frequency];
                 }
 
-                if ($this->shouldRunNow($frequency, $serverTimezone)) {
+                if (shouldRunCronNow($frequency, $serverTimezone, "scheduled-backup:{$backup->id}", $this->executionTime)) {
                     DatabaseBackupJob::dispatch($backup);
+                    $this->dispatchedCount++;
+                    Log::channel('scheduled')->info('Backup dispatched', [
+                        'backup_id' => $backup->id,
+                        'database_id' => $backup->database_id,
+                        'database_type' => $backup->database_type,
+                        'team_id' => $backup->team_id ?? null,
+                        'server_id' => $server->id,
+                    ]);
                 }
             } catch (\Exception $e) {
                 Log::channel('scheduled-errors')->error('Error processing backup', [
@@ -138,11 +212,21 @@ class ScheduledJobManager implements ShouldQueue
 
         foreach ($tasks as $task) {
             try {
-                if (! $this->shouldProcessTask($task)) {
+                $server = $task->server();
+
+                // Phase 1: Critical checks (always — cheap, handles orphans and infra issues)
+                $criticalSkip = $this->getTaskCriticalSkipReason($task, $server);
+                if ($criticalSkip !== null) {
+                    $this->skippedCount++;
+                    $this->logSkip('task', $criticalSkip, [
+                        'task_id' => $task->id,
+                        'task_name' => $task->name,
+                        'team_id' => $server?->team_id,
+                    ]);
+
                     continue;
                 }
 
-                $server = $task->server();
                 $serverTimezone = data_get($server->settings, 'server_timezone', config('app.timezone'));
 
                 if (validate_timezone($serverTimezone) === false) {
@@ -154,9 +238,31 @@ class ScheduledJobManager implements ShouldQueue
                     $frequency = VALID_CRON_STRINGS[$frequency];
                 }
 
-                if ($this->shouldRunNow($frequency, $serverTimezone)) {
-                    ScheduledTaskJob::dispatch($task);
+                if (! shouldRunCronNow($frequency, $serverTimezone, "scheduled-task:{$task->id}", $this->executionTime)) {
+                    continue;
                 }
+
+                // Phase 2: Runtime checks (only when cron is due — avoids noise for stopped resources)
+                $runtimeSkip = $this->getTaskRuntimeSkipReason($task);
+                if ($runtimeSkip !== null) {
+                    $this->skippedCount++;
+                    $this->logSkip('task', $runtimeSkip, [
+                        'task_id' => $task->id,
+                        'task_name' => $task->name,
+                        'team_id' => $server->team_id,
+                    ]);
+
+                    continue;
+                }
+
+                ScheduledTaskJob::dispatch($task);
+                $this->dispatchedCount++;
+                Log::channel('scheduled')->info('Task dispatched', [
+                    'task_id' => $task->id,
+                    'task_name' => $task->name,
+                    'team_id' => $server->team_id,
+                    'server_id' => $server->id,
+                ]);
             } catch (\Exception $e) {
                 Log::channel('scheduled-errors')->error('Error processing task', [
                     'task_id' => $task->id,
@@ -166,79 +272,67 @@ class ScheduledJobManager implements ShouldQueue
         }
     }
 
-    private function shouldProcessBackup(ScheduledDatabaseBackup $backup): bool
+    private function getBackupSkipReason(ScheduledDatabaseBackup $backup, ?Server $server): ?string
     {
         if (blank(data_get($backup, 'database'))) {
             $backup->delete();
 
-            return false;
+            return 'database_deleted';
         }
 
-        $server = $backup->server();
         if (blank($server)) {
             $backup->delete();
 
-            return false;
+            return 'server_deleted';
         }
 
         if ($server->isFunctional() === false) {
-            return false;
+            return 'server_not_functional';
         }
 
         if (isCloud() && data_get($server->team->subscription, 'stripe_invoice_paid', false) === false && $server->team->id !== 0) {
-            return false;
+            return 'subscription_unpaid';
         }
 
-        return true;
+        return null;
     }
 
-    private function shouldProcessTask(ScheduledTask $task): bool
+    private function getTaskCriticalSkipReason(ScheduledTask $task, ?Server $server): ?string
     {
-        $service = $task->service;
-        $application = $task->application;
-
-        $server = $task->server();
         if (blank($server)) {
             $task->delete();
 
-            return false;
+            return 'server_deleted';
         }
 
         if ($server->isFunctional() === false) {
-            return false;
+            return 'server_not_functional';
         }
 
         if (isCloud() && data_get($server->team->subscription, 'stripe_invoice_paid', false) === false && $server->team->id !== 0) {
-            return false;
+            return 'subscription_unpaid';
         }
 
-        if (! $service && ! $application) {
+        if (! $task->service && ! $task->application) {
             $task->delete();
 
-            return false;
+            return 'resource_deleted';
         }
 
-        if ($application && str($application->status)->contains('running') === false) {
-            return false;
-        }
-
-        if ($service && str($service->status)->contains('running') === false) {
-            return false;
-        }
-
-        return true;
+        return null;
     }
 
-    private function shouldRunNow(string $frequency, string $timezone): bool
+    private function getTaskRuntimeSkipReason(ScheduledTask $task): ?string
     {
-        $cron = new CronExpression($frequency);
+        if ($task->application && str($task->application->status)->contains('running') === false) {
+            return 'application_not_running';
+        }
 
-        // Use the frozen execution time, not the current time
-        // Fallback to current time if execution time is not set (shouldn't happen)
-        $baseTime = $this->executionTime ?? Carbon::now();
-        $executionTime = $baseTime->copy()->setTimezone($timezone);
+        if ($task->service && str($task->service->status)->contains('running') === false) {
+            return 'service_not_running';
+        }
 
-        return $cron->isDue($executionTime);
+        return null;
     }
 
     private function processDockerCleanups(): void
@@ -248,7 +342,15 @@ class ScheduledJobManager implements ShouldQueue
 
         foreach ($servers as $server) {
             try {
-                if (! $this->shouldProcessDockerCleanup($server)) {
+                $skipReason = $this->getDockerCleanupSkipReason($server);
+                if ($skipReason !== null) {
+                    $this->skippedCount++;
+                    $this->logSkip('docker_cleanup', $skipReason, [
+                        'server_id' => $server->id,
+                        'server_name' => $server->name,
+                        'team_id' => $server->team_id,
+                    ]);
+
                     continue;
                 }
 
@@ -263,13 +365,19 @@ class ScheduledJobManager implements ShouldQueue
                 }
 
                 // Use the frozen execution time for consistent evaluation
-                if ($this->shouldRunNow($frequency, $serverTimezone)) {
+                if (shouldRunCronNow($frequency, $serverTimezone, "docker-cleanup:{$server->id}", $this->executionTime)) {
                     DockerCleanupJob::dispatch(
                         $server,
                         false,
                         $server->settings->delete_unused_volumes,
                         $server->settings->delete_unused_networks
                     );
+                    $this->dispatchedCount++;
+                    Log::channel('scheduled')->info('Docker cleanup dispatched', [
+                        'server_id' => $server->id,
+                        'server_name' => $server->name,
+                        'team_id' => $server->team_id,
+                    ]);
                 }
             } catch (\Exception $e) {
                 Log::channel('scheduled-errors')->error('Error processing docker cleanup', [
@@ -296,19 +404,28 @@ class ScheduledJobManager implements ShouldQueue
         return $query->get();
     }
 
-    private function shouldProcessDockerCleanup(Server $server): bool
+    private function getDockerCleanupSkipReason(Server $server): ?string
     {
         if (! $server->isFunctional()) {
-            return false;
+            return 'server_not_functional';
         }
 
         // In cloud, check subscription status (except team 0)
         if (isCloud() && $server->team_id !== 0) {
             if (data_get($server->team->subscription, 'stripe_invoice_paid', false) === false) {
-                return false;
+                return 'subscription_unpaid';
             }
         }
 
-        return true;
+        return null;
+    }
+
+    private function logSkip(string $type, string $reason, array $context = []): void
+    {
+        Log::channel('scheduled')->info(ucfirst(str_replace('_', ' ', $type)).' skipped', array_merge([
+            'type' => $type,
+            'skip_reason' => $reason,
+            'execution_time' => $this->executionTime?->toIso8601String(),
+        ], $context));
     }
 }

app/Jobs/ScheduledTaskJob.php

@@ -14,13 +14,14 @@ use App\Notifications\ScheduledTask\TaskFailed;
 use App\Notifications\ScheduledTask\TaskSuccess;
 use Carbon\Carbon;
 use Illuminate\Bus\Queueable;
+use Illuminate\Contracts\Queue\ShouldBeEncrypted;
 use Illuminate\Contracts\Queue\ShouldQueue;
 use Illuminate\Foundation\Bus\Dispatchable;
 use Illuminate\Queue\InteractsWithQueue;
 use Illuminate\Queue\SerializesModels;
 use Illuminate\Support\Facades\Log;
 
-class ScheduledTaskJob implements ShouldQueue
+class ScheduledTaskJob implements ShouldBeEncrypted, ShouldQueue
 {
     use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
 

app/Jobs/SendMessageToSlackJob.php

@@ -4,16 +4,27 @@ namespace App\Jobs;
 
 use App\Notifications\Dto\SlackMessage;
 use Illuminate\Bus\Queueable;
+use Illuminate\Contracts\Queue\ShouldBeEncrypted;
 use Illuminate\Contracts\Queue\ShouldQueue;
 use Illuminate\Foundation\Bus\Dispatchable;
 use Illuminate\Queue\InteractsWithQueue;
 use Illuminate\Queue\SerializesModels;
 use Illuminate\Support\Facades\Http;
 
-class SendMessageToSlackJob implements ShouldQueue
+class SendMessageToSlackJob implements ShouldBeEncrypted, ShouldQueue
 {
     use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
 
+    /**
+     * The number of times the job may be attempted.
+     */
+    public $tries = 5;
+
+    /**
+     * The number of seconds to wait before retrying the job.
+     */
+    public $backoff = 10;
+
     public function __construct(
         private SlackMessage $message,
         private string $webhookUrl
@@ -22,6 +33,36 @@ class SendMessageToSlackJob implements ShouldQueue
     }
 
     public function handle(): void
+    {
+        if ($this->isSlackWebhook()) {
+            $this->sendToSlack();
+
+            return;
+        }
+
+        /**
+         * This works with Mattermost and as a fallback also with Slack, the notifications just look slightly different and advanced formatting for slack is not supported with Mattermost.
+         *
+         * @see https://github.com/coollabsio/coolify/pull/6139#issuecomment-3756777708
+         */
+        $this->sendToMattermost();
+    }
+
+    private function isSlackWebhook(): bool
+    {
+        $parsedUrl = parse_url($this->webhookUrl);
+
+        if ($parsedUrl === false) {
+            return false;
+        }
+
+        $scheme = $parsedUrl['scheme'] ?? '';
+        $host = $parsedUrl['host'] ?? '';
+
+        return $scheme === 'https' && $host === 'hooks.slack.com';
+    }
+
+    private function sendToSlack(): void
     {
         Http::post($this->webhookUrl, [
             'text' => $this->message->title,
@@ -57,4 +98,24 @@ class SendMessageToSlackJob implements ShouldQueue
             ],
         ]);
     }
+
+    /**
+     * @todo v5 refactor: Extract this into a separate SendMessageToMattermostJob.php triggered via the "mattermost" notification channel type.
+     */
+    private function sendToMattermost(): void
+    {
+        $username = config('app.name');
+
+        Http::post($this->webhookUrl, [
+            'username' => $username,
+            'attachments' => [
+                [
+                    'title' => $this->message->title,
+                    'color' => $this->message->color,
+                    'text' => $this->message->description,
+                    'footer' => $username,
+                ],
+            ],
+        ]);
+    }
 }

app/Jobs/SendMessageToTelegramJob.php

@@ -22,6 +22,11 @@ class SendMessageToTelegramJob implements ShouldBeEncrypted, ShouldQueue
      */
     public $tries = 5;
 
+    /**
+     * The number of seconds to wait before retrying the job.
+     */
+    public $backoff = 10;
+
     /**
      * The maximum number of unhandled exceptions to allow before failing.
      */

app/Jobs/SendWebhookJob.php

@@ -9,6 +9,8 @@ use Illuminate\Foundation\Bus\Dispatchable;
 use Illuminate\Queue\InteractsWithQueue;
 use Illuminate\Queue\SerializesModels;
 use Illuminate\Support\Facades\Http;
+use Illuminate\Support\Facades\Log;
+use Illuminate\Support\Facades\Validator;
 
 class SendWebhookJob implements ShouldBeEncrypted, ShouldQueue
 {
@@ -40,6 +42,20 @@ class SendWebhookJob implements ShouldBeEncrypted, ShouldQueue
      */
     public function handle(): void
     {
+        $validator = Validator::make(
+            ['webhook_url' => $this->webhookUrl],
+            ['webhook_url' => ['required', 'url', new \App\Rules\SafeWebhookUrl]]
+        );
+
+        if ($validator->fails()) {
+            Log::warning('SendWebhookJob: blocked unsafe webhook URL', [
+                'url' => $this->webhookUrl,
+                'errors' => $validator->errors()->all(),
+            ]);
+
+            return;
+        }
+
         if (isDev()) {
             ray('Sending webhook notification', [
                 'url' => $this->webhookUrl,

app/Jobs/ServerCheckJob.php

@@ -15,6 +15,7 @@ use Illuminate\Foundation\Bus\Dispatchable;
 use Illuminate\Queue\InteractsWithQueue;
 use Illuminate\Queue\Middleware\WithoutOverlapping;
 use Illuminate\Queue\SerializesModels;
+use Illuminate\Support\Facades\Log;
 
 class ServerCheckJob implements ShouldBeEncrypted, ShouldQueue
 {
@@ -33,6 +34,19 @@ class ServerCheckJob implements ShouldBeEncrypted, ShouldQueue
 
     public function __construct(public Server $server) {}
 
+    public function failed(?\Throwable $exception): void
+    {
+        if ($exception instanceof \Illuminate\Queue\TimeoutExceededException) {
+            Log::warning('ServerCheckJob timed out', [
+                'server_id' => $this->server->id,
+                'server_name' => $this->server->name,
+            ]);
+
+            // Delete the queue job so it doesn't appear in Horizon's failed list.
+            $this->job?->delete();
+        }
+    }
+
     public function handle()
     {
         try {

app/Jobs/ServerConnectionCheckJob.php

@@ -101,22 +101,34 @@ class ServerConnectionCheckJob implements ShouldBeEncrypted, ShouldQueue
                 'is_usable' => false,
             ]);
 
-            throw $e;
+            return;
+        }
+    }
+
+    public function failed(?\Throwable $exception): void
+    {
+        if ($exception instanceof \Illuminate\Queue\TimeoutExceededException) {
+            $this->server->settings->update([
+                'is_reachable' => false,
+                'is_usable' => false,
+            ]);
+
+            // Delete the queue job so it doesn't appear in Horizon's failed list.
+            $this->job?->delete();
         }
     }
 
     private function checkHetznerStatus(): void
     {
+        $status = null;
+
         try {
             $hetznerService = new \App\Services\HetznerService($this->server->cloudProviderToken->token);
             $serverData = $hetznerService->getServer($this->server->hetzner_server_id);
             $status = $serverData['status'] ?? null;
 
-        } catch (\Throwable $e) {
-            Log::debug('ServerConnectionCheck: Hetzner status check failed', [
-                'server_id' => $this->server->id,
-                'error' => $e->getMessage(),
-            ]);
+        } catch (\Throwable) {
+            // Silently ignore — server may have been deleted from Hetzner.
         }
         if ($this->server->hetzner_server_status !== $status) {
             $this->server->update(['hetzner_server_status' => $status]);

app/Jobs/ServerLimitCheckJob.php

@@ -38,7 +38,7 @@ class ServerLimitCheckJob implements ShouldBeEncrypted, ShouldQueue
                     $server->forceDisableServer();
                     $this->team->notify(new ForceDisabled($server));
                 });
-            } elseif ($number_of_servers_to_disable === 0) {
+            } elseif ($number_of_servers_to_disable <= 0) {
                 $servers->each(function ($server) {
                     if ($server->isForceDisabled()) {
                         $server->forceEnableServer();

app/Jobs/ServerManagerJob.php

@@ -5,8 +5,8 @@ namespace App\Jobs;
 use App\Models\InstanceSettings;
 use App\Models\Server;
 use App\Models\Team;
-use Cron\CronExpression;
 use Illuminate\Bus\Queueable;
+use Illuminate\Contracts\Queue\ShouldBeEncrypted;
 use Illuminate\Contracts\Queue\ShouldQueue;
 use Illuminate\Foundation\Bus\Dispatchable;
 use Illuminate\Queue\InteractsWithQueue;
@@ -15,7 +15,7 @@ use Illuminate\Support\Carbon;
 use Illuminate\Support\Collection;
 use Illuminate\Support\Facades\Log;
 
-class ServerManagerJob implements ShouldQueue
+class ServerManagerJob implements ShouldBeEncrypted, ShouldQueue
 {
     use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
 
@@ -64,11 +64,11 @@ class ServerManagerJob implements ShouldQueue
 
     private function getServers(): Collection
     {
-        $allServers = Server::where('ip', '!=', '1.2.3.4');
+        $allServers = Server::with('settings')->where('ip', '!=', '1.2.3.4');
 
         if (isCloud()) {
             $servers = $allServers->whereRelation('team.subscription', 'stripe_invoice_paid', true)->get();
-            $own = Team::find(0)->servers;
+            $own = Team::find(0)->servers()->with('settings')->get();
 
             return $servers->merge($own);
         } else {
@@ -79,9 +79,13 @@ class ServerManagerJob implements ShouldQueue
     private function dispatchConnectionChecks(Collection $servers): void
     {
 
-        if ($this->shouldRunNow($this->checkFrequency)) {
+        if (shouldRunCronNow($this->checkFrequency, $this->instanceTimezone, 'server-connection-checks', $this->executionTime)) {
             $servers->each(function (Server $server) {
                 try {
+                    // Skip SSH connection check if Sentinel is healthy — its heartbeat already proves connectivity
+                    if ($server->isSentinelEnabled() && $server->isSentinelLive()) {
+                        return;
+                    }
                     ServerConnectionCheckJob::dispatch($server);
                 } catch (\Exception $e) {
                     Log::channel('scheduled-errors')->error('Failed to dispatch ServerConnectionCheck', [
@@ -124,19 +128,17 @@ class ServerManagerJob implements ShouldQueue
 
         if ($sentinelOutOfSync) {
             // Dispatch ServerCheckJob if Sentinel is out of sync
-            if ($this->shouldRunNow($this->checkFrequency, $serverTimezone)) {
+            if (shouldRunCronNow($this->checkFrequency, $serverTimezone, "server-check:{$server->id}", $this->executionTime)) {
                 ServerCheckJob::dispatch($server);
             }
         }
 
         $isSentinelEnabled = $server->isSentinelEnabled();
-        $shouldRestartSentinel = $isSentinelEnabled && $this->shouldRunNow('0 0 * * *', $serverTimezone);
+        $shouldRestartSentinel = $isSentinelEnabled && shouldRunCronNow('0 0 * * *', $serverTimezone, "sentinel-restart:{$server->id}", $this->executionTime);
         // Dispatch Sentinel restart if due (daily for Sentinel-enabled servers)
 
         if ($shouldRestartSentinel) {
-            dispatch(function () use ($server) {
-                $server->restartContainer('coolify-sentinel');
-            });
+            CheckAndStartSentinelJob::dispatch($server);
         }
 
         // Dispatch ServerStorageCheckJob if due (only when Sentinel is out of sync or disabled)
@@ -146,7 +148,7 @@ class ServerManagerJob implements ShouldQueue
             if (isset(VALID_CRON_STRINGS[$serverDiskUsageCheckFrequency])) {
                 $serverDiskUsageCheckFrequency = VALID_CRON_STRINGS[$serverDiskUsageCheckFrequency];
             }
-            $shouldRunStorageCheck = $this->shouldRunNow($serverDiskUsageCheckFrequency, $serverTimezone);
+            $shouldRunStorageCheck = shouldRunCronNow($serverDiskUsageCheckFrequency, $serverTimezone, "server-storage-check:{$server->id}", $this->executionTime);
 
             if ($shouldRunStorageCheck) {
                 ServerStorageCheckJob::dispatch($server);
@@ -154,27 +156,13 @@ class ServerManagerJob implements ShouldQueue
         }
 
         // Dispatch ServerPatchCheckJob if due (weekly)
-        $shouldRunPatchCheck = $this->shouldRunNow('0 0 * * 0', $serverTimezone);
+        $shouldRunPatchCheck = shouldRunCronNow('0 0 * * 0', $serverTimezone, "server-patch-check:{$server->id}", $this->executionTime);
 
         if ($shouldRunPatchCheck) { // Weekly on Sunday at midnight
             ServerPatchCheckJob::dispatch($server);
         }
 
-        // Sentinel update checks (hourly) - check for updates to Sentinel version
-        // No timezone needed for hourly - runs at top of every hour
-        if ($isSentinelEnabled && $this->shouldRunNow('0 * * * *')) {
-            CheckAndStartSentinelJob::dispatch($server);
-        }
-    }
-
-    private function shouldRunNow(string $frequency, ?string $timezone = null): bool
-    {
-        $cron = new CronExpression($frequency);
-
-        // Use the frozen execution time, not the current time
-        $baseTime = $this->executionTime ?? Carbon::now();
-        $executionTime = $baseTime->copy()->setTimezone($timezone ?? config('app.timezone'));
-
-        return $cron->isDue($executionTime);
+        // Note: CheckAndStartSentinelJob is only dispatched daily (line above) for version updates.
+        // Crash recovery is handled by sentinelOutOfSync → ServerCheckJob → CheckAndStartSentinelJob.
     }
 }

app/Jobs/ServerStorageCheckJob.php

@@ -10,6 +10,7 @@ use Illuminate\Contracts\Queue\ShouldQueue;
 use Illuminate\Foundation\Bus\Dispatchable;
 use Illuminate\Queue\InteractsWithQueue;
 use Illuminate\Queue\SerializesModels;
+use Illuminate\Support\Facades\Log;
 use Illuminate\Support\Facades\RateLimiter;
 use Laravel\Horizon\Contracts\Silenced;
 
@@ -28,6 +29,19 @@ class ServerStorageCheckJob implements ShouldBeEncrypted, ShouldQueue, Silenced
 
     public function __construct(public Server $server, public int|string|null $percentage = null) {}
 
+    public function failed(?\Throwable $exception): void
+    {
+        if ($exception instanceof \Illuminate\Queue\TimeoutExceededException) {
+            Log::warning('ServerStorageCheckJob timed out', [
+                'server_id' => $this->server->id,
+                'server_name' => $this->server->name,
+            ]);
+
+            // Delete the queue job so it doesn't appear in Horizon's failed list.
+            $this->job?->delete();
+        }
+    }
+
     public function handle()
     {
         try {

app/Jobs/StripeProcessJob.php

@@ -2,13 +2,15 @@
 
 namespace App\Jobs;
 
+use App\Actions\Stripe\UpdateSubscriptionQuantity;
 use App\Models\Subscription;
 use App\Models\Team;
+use Illuminate\Contracts\Queue\ShouldBeEncrypted;
 use Illuminate\Contracts\Queue\ShouldQueue;
 use Illuminate\Foundation\Queue\Queueable;
 use Illuminate\Support\Str;
 
-class StripeProcessJob implements ShouldQueue
+class StripeProcessJob implements ShouldBeEncrypted, ShouldQueue
 {
     use Queueable;
 
@@ -71,25 +73,15 @@ class StripeProcessJob implements ShouldQueue
                         // send_internal_notification("User {$userId} is not an admin or owner of team {$team->id}, customerid: {$customerId}, subscriptionid: {$subscriptionId}.");
                         throw new \RuntimeException("User {$userId} is not an admin or owner of team {$team->id}, customerid: {$customerId}, subscriptionid: {$subscriptionId}.");
                     }
-                    $subscription = Subscription::where('team_id', $teamId)->first();
-                    if ($subscription) {
-                        // send_internal_notification('Old subscription activated for team: '.$teamId);
-                        $subscription->update([
+                    Subscription::updateOrCreate(
+                        ['team_id' => $teamId],
+                        [
                             'stripe_subscription_id' => $subscriptionId,
                             'stripe_customer_id' => $customerId,
                             'stripe_invoice_paid' => true,
                             'stripe_past_due' => false,
-                        ]);
-                    } else {
-                        // send_internal_notification('New subscription for team: '.$teamId);
-                        Subscription::create([
-                            'team_id' => $teamId,
-                            'stripe_subscription_id' => $subscriptionId,
-                            'stripe_customer_id' => $customerId,
-                            'stripe_invoice_paid' => true,
-                            'stripe_past_due' => false,
-                        ]);
-                    }
+                        ]
+                    );
                     break;
                 case 'invoice.paid':
                     $customerId = data_get($data, 'customer');
@@ -225,18 +217,15 @@ class StripeProcessJob implements ShouldQueue
                         // send_internal_notification("User {$userId} is not an admin or owner of team {$team->id}, customerid: {$customerId}.");
                         throw new \RuntimeException("User {$userId} is not an admin or owner of team {$team->id}, customerid: {$customerId}.");
                     }
-                    $subscription = Subscription::where('team_id', $teamId)->first();
-                    if ($subscription) {
-                        // send_internal_notification("Subscription already exists for team: {$teamId}");
-                        throw new \RuntimeException("Subscription already exists for team: {$teamId}");
-                    } else {
-                        Subscription::create([
-                            'team_id' => $teamId,
+                    Subscription::updateOrCreate(
+                        ['team_id' => $teamId],
+                        [
                             'stripe_subscription_id' => $subscriptionId,
                             'stripe_customer_id' => $customerId,
                             'stripe_invoice_paid' => false,
-                        ]);
-                    }
+                        ]
+                    );
+                    break;
                 case 'customer.subscription.updated':
                     $teamId = data_get($data, 'metadata.team_id');
                     $userId = data_get($data, 'metadata.user_id');
@@ -251,34 +240,33 @@ class StripeProcessJob implements ShouldQueue
                     $subscription = Subscription::where('stripe_customer_id', $customerId)->first();
                     if (! $subscription) {
                         if ($status === 'incomplete_expired') {
-                            // send_internal_notification('Subscription incomplete expired');
                             throw new \RuntimeException('Subscription incomplete expired');
                         }
-                        if ($teamId) {
-                            $subscription = Subscription::create([
-                                'team_id' => $teamId,
+                        if (! $teamId) {
+                            throw new \RuntimeException('No subscription and team id found');
+                        }
+                        $subscription = Subscription::firstOrCreate(
+                            ['team_id' => $teamId],
+                            [
                                 'stripe_subscription_id' => $subscriptionId,
                                 'stripe_customer_id' => $customerId,
                                 'stripe_invoice_paid' => false,
-                            ]);
-                        } else {
-                            // send_internal_notification('No subscription and team id found');
-                            throw new \RuntimeException('No subscription and team id found');
-                        }
+                            ]
+                        );
                     }
                     $cancelAtPeriodEnd = data_get($data, 'cancel_at_period_end');
                     $feedback = data_get($data, 'cancellation_details.feedback');
                     $comment = data_get($data, 'cancellation_details.comment');
                     $lookup_key = data_get($data, 'items.data.0.price.lookup_key');
                     if (str($lookup_key)->contains('dynamic')) {
-                        $quantity = data_get($data, 'items.data.0.quantity', 2);
+                        $quantity = min((int) data_get($data, 'items.data.0.quantity', 2), UpdateSubscriptionQuantity::MAX_SERVER_LIMIT);
                         $team = data_get($subscription, 'team');
                         if ($team) {
                             $team->update([
                                 'custom_server_limit' => $quantity,
                             ]);
+                            ServerLimitCheckJob::dispatch($team);
                         }
-                        ServerLimitCheckJob::dispatch($team);
                     }
                     $subscription->update([
                         'stripe_feedback' => $feedback,

app/Jobs/SyncStripeSubscriptionsJob.php

@@ -4,12 +4,13 @@ namespace App\Jobs;
 
 use App\Models\Subscription;
 use Illuminate\Bus\Queueable;
+use Illuminate\Contracts\Queue\ShouldBeEncrypted;
 use Illuminate\Contracts\Queue\ShouldQueue;
 use Illuminate\Foundation\Bus\Dispatchable;
 use Illuminate\Queue\InteractsWithQueue;
 use Illuminate\Queue\SerializesModels;
 
-class SyncStripeSubscriptionsJob implements ShouldQueue
+class SyncStripeSubscriptionsJob implements ShouldBeEncrypted, ShouldQueue
 {
     use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
 
@@ -22,7 +23,7 @@ class SyncStripeSubscriptionsJob implements ShouldQueue
         $this->onQueue('high');
     }
 
-    public function handle(): array
+    public function handle(?\Closure $onProgress = null): array
     {
         if (! isCloud() || ! isStripe()) {
             return ['error' => 'Not running on Cloud or Stripe not configured'];
@@ -33,48 +34,73 @@ class SyncStripeSubscriptionsJob implements ShouldQueue
             ->get();
 
         $stripe = new \Stripe\StripeClient(config('subscription.stripe_api_key'));
+
+        // Bulk fetch all valid subscription IDs from Stripe (active + past_due)
+        $validStripeIds = $this->fetchValidStripeSubscriptionIds($stripe, $onProgress);
+
+        // Find DB subscriptions not in the valid set
+        $staleSubscriptions = $subscriptions->filter(
+            fn (Subscription $sub) => ! in_array($sub->stripe_subscription_id, $validStripeIds)
+        );
+
+        // For each stale subscription, get the exact Stripe status and check for resubscriptions
         $discrepancies = [];
+        $resubscribed = [];
         $errors = [];
 
-        foreach ($subscriptions as $subscription) {
+        foreach ($staleSubscriptions as $subscription) {
             try {
                 $stripeSubscription = $stripe->subscriptions->retrieve(
                     $subscription->stripe_subscription_id
                 );
+                $stripeStatus = $stripeSubscription->status;
 
-                // Check if Stripe says cancelled but we think it's active
-                if (in_array($stripeSubscription->status, ['canceled', 'incomplete_expired', 'unpaid'])) {
-                    $discrepancies[] = [
-                        'subscription_id' => $subscription->id,
-                        'team_id' => $subscription->team_id,
-                        'stripe_subscription_id' => $subscription->stripe_subscription_id,
-                        'stripe_status' => $stripeSubscription->status,
-                    ];
-
-                    // Only fix if --fix flag is passed
-                    if ($this->fix) {
-                        $subscription->update([
-                            'stripe_invoice_paid' => false,
-                            'stripe_past_due' => false,
-                        ]);
-
-                        if ($stripeSubscription->status === 'canceled') {
-                            $subscription->team?->subscriptionEnded();
-                        }
-                    }
-                }
-
-                // Small delay to avoid Stripe rate limits
-                usleep(100000); // 100ms
+                usleep(100000); // 100ms rate limit delay
             } catch (\Exception $e) {
                 $errors[] = [
                     'subscription_id' => $subscription->id,
                     'error' => $e->getMessage(),
                 ];
+
+                continue;
+            }
+
+            // Check if this user resubscribed under a different customer/subscription
+            $activeSub = $this->findActiveSubscriptionByEmail($stripe, $stripeSubscription->customer);
+            if ($activeSub) {
+                $resubscribed[] = [
+                    'subscription_id' => $subscription->id,
+                    'team_id' => $subscription->team_id,
+                    'email' => $activeSub['email'],
+                    'old_stripe_subscription_id' => $subscription->stripe_subscription_id,
+                    'old_stripe_customer_id' => $stripeSubscription->customer,
+                    'new_stripe_subscription_id' => $activeSub['subscription_id'],
+                    'new_stripe_customer_id' => $activeSub['customer_id'],
+                    'new_status' => $activeSub['status'],
+                ];
+
+                continue;
+            }
+
+            $discrepancies[] = [
+                'subscription_id' => $subscription->id,
+                'team_id' => $subscription->team_id,
+                'stripe_subscription_id' => $subscription->stripe_subscription_id,
+                'stripe_status' => $stripeStatus,
+            ];
+
+            if ($this->fix) {
+                $subscription->update([
+                    'stripe_invoice_paid' => false,
+                    'stripe_past_due' => false,
+                ]);
+
+                if ($stripeStatus === 'canceled') {
+                    $subscription->team?->subscriptionEnded();
+                }
             }
         }
 
-        // Only notify if discrepancies found and fixed
         if ($this->fix && count($discrepancies) > 0) {
             send_internal_notification(
                 'SyncStripeSubscriptionsJob: Fixed '.count($discrepancies)." discrepancies:\n".
@@ -85,8 +111,88 @@ class SyncStripeSubscriptionsJob implements ShouldQueue
         return [
             'total_checked' => $subscriptions->count(),
             'discrepancies' => $discrepancies,
+            'resubscribed' => $resubscribed,
             'errors' => $errors,
             'fixed' => $this->fix,
         ];
     }
+
+    /**
+     * Given a Stripe customer ID, get their email and search for other customers
+     * with the same email that have an active subscription.
+     *
+     * @return array{email: string, customer_id: string, subscription_id: string, status: string}|null
+     */
+    private function findActiveSubscriptionByEmail(\Stripe\StripeClient $stripe, string $customerId): ?array
+    {
+        try {
+            $customer = $stripe->customers->retrieve($customerId);
+            $email = $customer->email;
+
+            if (! $email) {
+                return null;
+            }
+
+            usleep(100000);
+
+            $customers = $stripe->customers->all([
+                'email' => $email,
+                'limit' => 10,
+            ]);
+
+            usleep(100000);
+
+            foreach ($customers->data as $matchingCustomer) {
+                if ($matchingCustomer->id === $customerId) {
+                    continue;
+                }
+
+                $subs = $stripe->subscriptions->all([
+                    'customer' => $matchingCustomer->id,
+                    'limit' => 10,
+                ]);
+
+                usleep(100000);
+
+                foreach ($subs->data as $sub) {
+                    if (in_array($sub->status, ['active', 'past_due'])) {
+                        return [
+                            'email' => $email,
+                            'customer_id' => $matchingCustomer->id,
+                            'subscription_id' => $sub->id,
+                            'status' => $sub->status,
+                        ];
+                    }
+                }
+            }
+        } catch (\Exception $e) {
+            // Silently skip — will fall through to normal discrepancy
+        }
+
+        return null;
+    }
+
+    /**
+     * Bulk fetch all active and past_due subscription IDs from Stripe.
+     *
+     * @return array<string>
+     */
+    private function fetchValidStripeSubscriptionIds(\Stripe\StripeClient $stripe, ?\Closure $onProgress = null): array
+    {
+        $validIds = [];
+        $fetched = 0;
+
+        foreach (['active', 'past_due'] as $status) {
+            foreach ($stripe->subscriptions->all(['status' => $status, 'limit' => 100])->autoPagingIterator() as $sub) {
+                $validIds[] = $sub->id;
+                $fetched++;
+
+                if ($onProgress) {
+                    $onProgress($fetched);
+                }
+            }
+        }
+
+        return $validIds;
+    }
 }

app/Jobs/ValidateAndInstallServerJob.php

@@ -8,13 +8,14 @@ use App\Events\ServerReachabilityChanged;
 use App\Events\ServerValidated;
 use App\Models\Server;
 use Illuminate\Bus\Queueable;
+use Illuminate\Contracts\Queue\ShouldBeEncrypted;
 use Illuminate\Contracts\Queue\ShouldQueue;
 use Illuminate\Foundation\Bus\Dispatchable;
 use Illuminate\Queue\InteractsWithQueue;
 use Illuminate\Queue\SerializesModels;
 use Illuminate\Support\Facades\Log;
 
-class ValidateAndInstallServerJob implements ShouldQueue
+class ValidateAndInstallServerJob implements ShouldBeEncrypted, ShouldQueue
 {
     use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
 
@@ -44,7 +45,8 @@ class ValidateAndInstallServerJob implements ShouldQueue
             // Validate connection
             ['uptime' => $uptime, 'error' => $error] = $this->server->validateConnection();
             if (! $uptime) {
-                $errorMessage = 'Server is not reachable. Please validate your configuration and connection.<br>Check this <a target="_blank" class="underline" href="https://coolify.io/docs/knowledge-base/server/openssh">documentation</a> for further help. <br><br>Error: '.$error;
+                $sanitizedError = htmlspecialchars($error ?? '', ENT_QUOTES, 'UTF-8');
+                $errorMessage = 'Server is not reachable. Please validate your configuration and connection.<br>Check this <a target="_blank" class="underline" href="https://coolify.io/docs/knowledge-base/server/openssh">documentation</a> for further help. <br><br>Error: '.$sanitizedError;
                 $this->server->update([
                     'validation_logs' => $errorMessage,
                     'is_validating' => false,
@@ -178,6 +180,9 @@ class ValidateAndInstallServerJob implements ShouldQueue
             // Mark validation as complete
             $this->server->update(['is_validating' => false]);
 
+            // Auto-fetch server details now that validation passed
+            $this->server->gatherServerMetadata();
+
             // Refresh server to get latest state
             $this->server->refresh();
 
@@ -193,7 +198,7 @@ class ValidateAndInstallServerJob implements ShouldQueue
             ]);
 
             $this->server->update([
-                'validation_logs' => 'An error occurred during validation: '.$e->getMessage(),
+                'validation_logs' => 'An error occurred during validation: '.htmlspecialchars($e->getMessage(), ENT_QUOTES, 'UTF-8'),
                 'is_validating' => false,
             ]);
         }

app/Jobs/VerifyStripeSubscriptionStatusJob.php

@@ -4,12 +4,13 @@ namespace App\Jobs;
 
 use App\Models\Subscription;
 use Illuminate\Bus\Queueable;
+use Illuminate\Contracts\Queue\ShouldBeEncrypted;
 use Illuminate\Contracts\Queue\ShouldQueue;
 use Illuminate\Foundation\Bus\Dispatchable;
 use Illuminate\Queue\InteractsWithQueue;
 use Illuminate\Queue\SerializesModels;
 
-class VerifyStripeSubscriptionStatusJob implements ShouldQueue
+class VerifyStripeSubscriptionStatusJob implements ShouldBeEncrypted, ShouldQueue
 {
     use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
 
@@ -81,12 +82,9 @@ class VerifyStripeSubscriptionStatusJob implements ShouldQueue
                         'stripe_past_due' => false,
                     ]);
 
-                    // Trigger subscription ended logic if canceled
-                    if ($stripeSubscription->status === 'canceled') {
-                        $team = $this->subscription->team;
-                        if ($team) {
-                            $team->subscriptionEnded();
-                        }
+                    $team = $this->subscription->team;
+                    if ($team) {
+                        $team->subscriptionEnded();
                     }
                     break;