Merge branch 'next' into feat/healthcheck-cmd

app/Jobs/ApplicationDeploymentJob.php

@@ -1208,7 +1208,7 @@ class ApplicationDeploymentJob implements ShouldBeEncrypted, ShouldQueue
                 }
                 $services = data_get($dockerCompose, 'services', []);
                 foreach ($services as $serviceName => $_) {
-                    $envs->push('SERVICE_NAME_'.str($serviceName)->upper().'='.$serviceName);
+                    $envs->push('SERVICE_NAME_'.str($serviceName)->replace('-', '_')->replace('.', '_')->upper().'='.$serviceName);
                 }
             }
 
@@ -1265,8 +1265,8 @@ class ApplicationDeploymentJob implements ShouldBeEncrypted, ShouldQueue
                         $coolifyScheme = $coolifyUrl->getScheme();
                         $coolifyFqdn = $coolifyUrl->getHost();
                         $coolifyUrl = $coolifyUrl->withScheme($coolifyScheme)->withHost($coolifyFqdn)->withPort(null);
-                        $envs->push('SERVICE_URL_'.str($forServiceName)->upper().'='.$coolifyUrl->__toString());
-                        $envs->push('SERVICE_FQDN_'.str($forServiceName)->upper().'='.$coolifyFqdn);
+                        $envs->push('SERVICE_URL_'.str($forServiceName)->replace('-', '_')->replace('.', '_')->upper().'='.$coolifyUrl->__toString());
+                        $envs->push('SERVICE_FQDN_'.str($forServiceName)->replace('-', '_')->replace('.', '_')->upper().'='.$coolifyFqdn);
                     }
                 }
 
@@ -1274,7 +1274,7 @@ class ApplicationDeploymentJob implements ShouldBeEncrypted, ShouldQueue
                 $rawDockerCompose = Yaml::parse($this->application->docker_compose_raw);
                 $rawServices = data_get($rawDockerCompose, 'services', []);
                 foreach ($rawServices as $rawServiceName => $_) {
-                    $envs->push('SERVICE_NAME_'.str($rawServiceName)->upper().'='.addPreviewDeploymentSuffix($rawServiceName, $this->pull_request_id));
+                    $envs->push('SERVICE_NAME_'.str($rawServiceName)->replace('-', '_')->replace('.', '_')->upper().'='.addPreviewDeploymentSuffix($rawServiceName, $this->pull_request_id));
                 }
             }
 
@@ -1476,7 +1476,7 @@ class ApplicationDeploymentJob implements ShouldBeEncrypted, ShouldQueue
                 }
                 $services = data_get($dockerCompose, 'services', []);
                 foreach ($services as $serviceName => $_) {
-                    $envs_dict['SERVICE_NAME_'.str($serviceName)->upper()] = escapeBashEnvValue($serviceName);
+                    $envs_dict['SERVICE_NAME_'.str($serviceName)->replace('-', '_')->replace('.', '_')->upper()] = escapeBashEnvValue($serviceName);
                 }
 
                 // Generate SERVICE_FQDN & SERVICE_URL for non-PR deployments
@@ -1489,8 +1489,8 @@ class ApplicationDeploymentJob implements ShouldBeEncrypted, ShouldQueue
                         $coolifyScheme = $coolifyUrl->getScheme();
                         $coolifyFqdn = $coolifyUrl->getHost();
                         $coolifyUrl = $coolifyUrl->withScheme($coolifyScheme)->withHost($coolifyFqdn)->withPort(null);
-                        $envs_dict['SERVICE_URL_'.str($forServiceName)->upper()] = escapeBashEnvValue($coolifyUrl->__toString());
-                        $envs_dict['SERVICE_FQDN_'.str($forServiceName)->upper()] = escapeBashEnvValue($coolifyFqdn);
+                        $envs_dict['SERVICE_URL_'.str($forServiceName)->replace('-', '_')->replace('.', '_')->upper()] = escapeBashEnvValue($coolifyUrl->__toString());
+                        $envs_dict['SERVICE_FQDN_'.str($forServiceName)->replace('-', '_')->replace('.', '_')->upper()] = escapeBashEnvValue($coolifyFqdn);
                     }
                 }
             } else {
@@ -1498,7 +1498,7 @@ class ApplicationDeploymentJob implements ShouldBeEncrypted, ShouldQueue
                 $rawDockerCompose = Yaml::parse($this->application->docker_compose_raw);
                 $rawServices = data_get($rawDockerCompose, 'services', []);
                 foreach ($rawServices as $rawServiceName => $_) {
-                    $envs_dict['SERVICE_NAME_'.str($rawServiceName)->upper()] = escapeBashEnvValue(addPreviewDeploymentSuffix($rawServiceName, $this->pull_request_id));
+                    $envs_dict['SERVICE_NAME_'.str($rawServiceName)->replace('-', '_')->replace('.', '_')->upper()] = escapeBashEnvValue(addPreviewDeploymentSuffix($rawServiceName, $this->pull_request_id));
                 }
 
                 // Generate SERVICE_FQDN & SERVICE_URL for preview deployments with PR-specific domains
@@ -1511,8 +1511,8 @@ class ApplicationDeploymentJob implements ShouldBeEncrypted, ShouldQueue
                         $coolifyScheme = $coolifyUrl->getScheme();
                         $coolifyFqdn = $coolifyUrl->getHost();
                         $coolifyUrl = $coolifyUrl->withScheme($coolifyScheme)->withHost($coolifyFqdn)->withPort(null);
-                        $envs_dict['SERVICE_URL_'.str($forServiceName)->upper()] = escapeBashEnvValue($coolifyUrl->__toString());
-                        $envs_dict['SERVICE_FQDN_'.str($forServiceName)->upper()] = escapeBashEnvValue($coolifyFqdn);
+                        $envs_dict['SERVICE_URL_'.str($forServiceName)->replace('-', '_')->replace('.', '_')->upper()] = escapeBashEnvValue($coolifyUrl->__toString());
+                        $envs_dict['SERVICE_FQDN_'.str($forServiceName)->replace('-', '_')->replace('.', '_')->upper()] = escapeBashEnvValue($coolifyFqdn);
                     }
                 }
             }

app/Jobs/ApplicationPullRequestUpdateJob.php

@@ -47,7 +47,7 @@ class ApplicationPullRequestUpdateJob implements ShouldBeEncrypted, ShouldQueue
             match ($this->status) {
                 ProcessStatus::QUEUED => $this->body = "The preview deployment for **{$serviceName}** is queued. ⏳\n\n",
                 ProcessStatus::IN_PROGRESS => $this->body = "The preview deployment for **{$serviceName}** is in progress. 🟡\n\n",
-                ProcessStatus::FINISHED => $this->body = "The preview deployment for **{$serviceName}** is ready. 🟢\n\n".($this->preview->fqdn ? "[Open Preview]({$this->preview->fqdn}) | " : ''),
+                ProcessStatus::FINISHED => $this->body = "The preview deployment for **{$serviceName}** is ready. 🟢\n\n".$this->getPreviewLinks(),
                 ProcessStatus::ERROR => $this->body = "The preview deployment for **{$serviceName}** failed. 🔴\n\n",
                 ProcessStatus::KILLED => $this->body = "The preview deployment for **{$serviceName}** was killed. ⚫\n\n",
                 ProcessStatus::CANCELLED => $this->body = "The preview deployment for **{$serviceName}** was cancelled. 🚫\n\n",
@@ -91,4 +91,27 @@ class ApplicationPullRequestUpdateJob implements ShouldBeEncrypted, ShouldQueue
     {
         githubApi(source: $this->application->source, endpoint: "/repos/{$this->application->git_repository}/issues/comments/{$this->preview->pull_request_issue_comment_id}", method: 'delete');
     }
+
+    private function getPreviewLinks(): string
+    {
+        if ($this->application->build_pack === 'dockercompose') {
+            $dockerComposeDomains = json_decode($this->preview->docker_compose_domains, true) ?? [];
+            $links = [];
+
+            foreach ($dockerComposeDomains as $serviceName => $config) {
+                $domain = data_get($config, 'domain');
+                if (! empty($domain)) {
+                    $firstDomain = str($domain)->explode(',')->first();
+                    $firstDomain = trim($firstDomain);
+                    if (! empty($firstDomain)) {
+                        $links[] = "[Open {$serviceName}]({$firstDomain})";
+                    }
+                }
+            }
+
+            return ! empty($links) ? implode(' | ', $links).' | ' : '';
+        }
+
+        return $this->preview->fqdn ? "[Open Preview]({$this->preview->fqdn}) | " : '';
+    }
 }

app/Jobs/PushServerUpdateJob.php

@@ -207,6 +207,9 @@ class PushServerUpdateJob implements ShouldBeEncrypted, ShouldQueue, Silenced
                 $serviceId = $labels->get('coolify.serviceId');
                 $subType = $labels->get('coolify.service.subType');
                 $subId = $labels->get('coolify.service.subId');
+                if (empty($subId)) {
+                    continue;
+                }
                 if ($subType === 'application') {
                     $this->foundServiceApplicationIds->push($subId);
                     // Store container status for aggregation
@@ -496,7 +499,14 @@ class PushServerUpdateJob implements ShouldBeEncrypted, ShouldQueue, Silenced
             if (! $tcpProxyContainerFound) {
                 StartDatabaseProxy::dispatch($database);
                 $this->server->team?->notify(new ContainerRestarted("TCP Proxy for {$database->name}", $this->server));
-            } else {
+            }
+        } elseif ($this->isRunning($containerStatus) && ! $tcpProxy) {
+            // Clean up orphaned proxy containers when is_public=false
+            $orphanedProxy = $this->containers->filter(function ($value, $key) use ($databaseUuid) {
+                return data_get($value, 'name') === "$databaseUuid-proxy" && data_get($value, 'state') === 'running';
+            })->first();
+            if ($orphanedProxy) {
+                StopDatabaseProxy::dispatch($database);
             }
         }
     }

app/Jobs/ScheduledJobManager.php

@@ -27,6 +27,10 @@ class ScheduledJobManager implements ShouldQueue
      */
     private ?Carbon $executionTime = null;
 
+    private int $dispatchedCount = 0;
+
+    private int $skippedCount = 0;
+
     /**
      * Create a new job instance.
      */
@@ -61,6 +65,12 @@ class ScheduledJobManager implements ShouldQueue
     {
         // Freeze the execution time at the start of the job
         $this->executionTime = Carbon::now();
+        $this->dispatchedCount = 0;
+        $this->skippedCount = 0;
+
+        Log::channel('scheduled')->info('ScheduledJobManager started', [
+            'execution_time' => $this->executionTime->toIso8601String(),
+        ]);
 
         // Process backups - don't let failures stop task processing
         try {
@@ -91,6 +101,13 @@ class ScheduledJobManager implements ShouldQueue
                 'trace' => $e->getTraceAsString(),
             ]);
         }
+
+        Log::channel('scheduled')->info('ScheduledJobManager completed', [
+            'execution_time' => $this->executionTime->toIso8601String(),
+            'duration_ms' => Carbon::now()->diffInMilliseconds($this->executionTime),
+            'dispatched' => $this->dispatchedCount,
+            'skipped' => $this->skippedCount,
+        ]);
     }
 
     private function processScheduledBackups(): void
@@ -101,8 +118,16 @@ class ScheduledJobManager implements ShouldQueue
 
         foreach ($backups as $backup) {
             try {
-                // Apply the same filtering logic as the original
-                if (! $this->shouldProcessBackup($backup)) {
+                $skipReason = $this->getBackupSkipReason($backup);
+                if ($skipReason !== null) {
+                    $this->skippedCount++;
+                    $this->logSkip('backup', $skipReason, [
+                        'backup_id' => $backup->id,
+                        'database_id' => $backup->database_id,
+                        'database_type' => $backup->database_type,
+                        'team_id' => $backup->team_id ?? null,
+                    ]);
+
                     continue;
                 }
 
@@ -120,6 +145,14 @@ class ScheduledJobManager implements ShouldQueue
 
                 if ($this->shouldRunNow($frequency, $serverTimezone)) {
                     DatabaseBackupJob::dispatch($backup);
+                    $this->dispatchedCount++;
+                    Log::channel('scheduled')->info('Backup dispatched', [
+                        'backup_id' => $backup->id,
+                        'database_id' => $backup->database_id,
+                        'database_type' => $backup->database_type,
+                        'team_id' => $backup->team_id ?? null,
+                        'server_id' => $server->id,
+                    ]);
                 }
             } catch (\Exception $e) {
                 Log::channel('scheduled-errors')->error('Error processing backup', [
@@ -138,7 +171,15 @@ class ScheduledJobManager implements ShouldQueue
 
         foreach ($tasks as $task) {
             try {
-                if (! $this->shouldProcessTask($task)) {
+                $skipReason = $this->getTaskSkipReason($task);
+                if ($skipReason !== null) {
+                    $this->skippedCount++;
+                    $this->logSkip('task', $skipReason, [
+                        'task_id' => $task->id,
+                        'task_name' => $task->name,
+                        'team_id' => $task->server()?->team_id,
+                    ]);
+
                     continue;
                 }
 
@@ -156,6 +197,13 @@ class ScheduledJobManager implements ShouldQueue
 
                 if ($this->shouldRunNow($frequency, $serverTimezone)) {
                     ScheduledTaskJob::dispatch($task);
+                    $this->dispatchedCount++;
+                    Log::channel('scheduled')->info('Task dispatched', [
+                        'task_id' => $task->id,
+                        'task_name' => $task->name,
+                        'team_id' => $server->team_id,
+                        'server_id' => $server->id,
+                    ]);
                 }
             } catch (\Exception $e) {
                 Log::channel('scheduled-errors')->error('Error processing task', [
@@ -166,33 +214,33 @@ class ScheduledJobManager implements ShouldQueue
         }
     }
 
-    private function shouldProcessBackup(ScheduledDatabaseBackup $backup): bool
+    private function getBackupSkipReason(ScheduledDatabaseBackup $backup): ?string
     {
         if (blank(data_get($backup, 'database'))) {
             $backup->delete();
 
-            return false;
+            return 'database_deleted';
         }
 
         $server = $backup->server();
         if (blank($server)) {
             $backup->delete();
 
-            return false;
+            return 'server_deleted';
         }
 
         if ($server->isFunctional() === false) {
-            return false;
+            return 'server_not_functional';
         }
 
         if (isCloud() && data_get($server->team->subscription, 'stripe_invoice_paid', false) === false && $server->team->id !== 0) {
-            return false;
+            return 'subscription_unpaid';
         }
 
-        return true;
+        return null;
     }
 
-    private function shouldProcessTask(ScheduledTask $task): bool
+    private function getTaskSkipReason(ScheduledTask $task): ?string
     {
         $service = $task->service;
         $application = $task->application;
@@ -201,32 +249,32 @@ class ScheduledJobManager implements ShouldQueue
         if (blank($server)) {
             $task->delete();
 
-            return false;
+            return 'server_deleted';
         }
 
         if ($server->isFunctional() === false) {
-            return false;
+            return 'server_not_functional';
         }
 
         if (isCloud() && data_get($server->team->subscription, 'stripe_invoice_paid', false) === false && $server->team->id !== 0) {
-            return false;
+            return 'subscription_unpaid';
         }
 
         if (! $service && ! $application) {
             $task->delete();
 
-            return false;
+            return 'resource_deleted';
         }
 
         if ($application && str($application->status)->contains('running') === false) {
-            return false;
+            return 'application_not_running';
         }
 
         if ($service && str($service->status)->contains('running') === false) {
-            return false;
+            return 'service_not_running';
         }
 
-        return true;
+        return null;
     }
 
     private function shouldRunNow(string $frequency, string $timezone): bool
@@ -248,7 +296,15 @@ class ScheduledJobManager implements ShouldQueue
 
         foreach ($servers as $server) {
             try {
-                if (! $this->shouldProcessDockerCleanup($server)) {
+                $skipReason = $this->getDockerCleanupSkipReason($server);
+                if ($skipReason !== null) {
+                    $this->skippedCount++;
+                    $this->logSkip('docker_cleanup', $skipReason, [
+                        'server_id' => $server->id,
+                        'server_name' => $server->name,
+                        'team_id' => $server->team_id,
+                    ]);
+
                     continue;
                 }
 
@@ -270,6 +326,12 @@ class ScheduledJobManager implements ShouldQueue
                         $server->settings->delete_unused_volumes,
                         $server->settings->delete_unused_networks
                     );
+                    $this->dispatchedCount++;
+                    Log::channel('scheduled')->info('Docker cleanup dispatched', [
+                        'server_id' => $server->id,
+                        'server_name' => $server->name,
+                        'team_id' => $server->team_id,
+                    ]);
                 }
             } catch (\Exception $e) {
                 Log::channel('scheduled-errors')->error('Error processing docker cleanup', [
@@ -296,19 +358,28 @@ class ScheduledJobManager implements ShouldQueue
         return $query->get();
     }
 
-    private function shouldProcessDockerCleanup(Server $server): bool
+    private function getDockerCleanupSkipReason(Server $server): ?string
     {
         if (! $server->isFunctional()) {
-            return false;
+            return 'server_not_functional';
         }
 
         // In cloud, check subscription status (except team 0)
         if (isCloud() && $server->team_id !== 0) {
             if (data_get($server->team->subscription, 'stripe_invoice_paid', false) === false) {
-                return false;
+                return 'subscription_unpaid';
             }
         }
 
-        return true;
+        return null;
+    }
+
+    private function logSkip(string $type, string $reason, array $context = []): void
+    {
+        Log::channel('scheduled')->info(ucfirst(str_replace('_', ' ', $type)).' skipped', array_merge([
+            'type' => $type,
+            'skip_reason' => $reason,
+            'execution_time' => $this->executionTime?->toIso8601String(),
+        ], $context));
     }
 }

app/Jobs/SendMessageToSlackJob.php

@@ -22,6 +22,36 @@ class SendMessageToSlackJob implements ShouldQueue
     }
 
     public function handle(): void
+    {
+        if ($this->isSlackWebhook()) {
+            $this->sendToSlack();
+
+            return;
+        }
+
+        /**
+         * This works with Mattermost and as a fallback also with Slack, the notifications just look slightly different and advanced formatting for slack is not supported with Mattermost.
+         *
+         * @see https://github.com/coollabsio/coolify/pull/6139#issuecomment-3756777708
+         */
+        $this->sendToMattermost();
+    }
+
+    private function isSlackWebhook(): bool
+    {
+        $parsedUrl = parse_url($this->webhookUrl);
+
+        if ($parsedUrl === false) {
+            return false;
+        }
+
+        $scheme = $parsedUrl['scheme'] ?? '';
+        $host = $parsedUrl['host'] ?? '';
+
+        return $scheme === 'https' && $host === 'hooks.slack.com';
+    }
+
+    private function sendToSlack(): void
     {
         Http::post($this->webhookUrl, [
             'text' => $this->message->title,
@@ -57,4 +87,24 @@ class SendMessageToSlackJob implements ShouldQueue
             ],
         ]);
     }
+
+    /**
+     * @todo v5 refactor: Extract this into a separate SendMessageToMattermostJob.php triggered via the "mattermost" notification channel type.
+     */
+    private function sendToMattermost(): void
+    {
+        $username = config('app.name');
+
+        Http::post($this->webhookUrl, [
+            'username' => $username,
+            'attachments' => [
+                [
+                    'title' => $this->message->title,
+                    'color' => $this->message->color,
+                    'text' => $this->message->description,
+                    'footer' => $username,
+                ],
+            ],
+        ]);
+    }
 }

app/Jobs/ServerCheckJob.php

@@ -15,6 +15,7 @@ use Illuminate\Foundation\Bus\Dispatchable;
 use Illuminate\Queue\InteractsWithQueue;
 use Illuminate\Queue\Middleware\WithoutOverlapping;
 use Illuminate\Queue\SerializesModels;
+use Illuminate\Support\Facades\Log;
 
 class ServerCheckJob implements ShouldBeEncrypted, ShouldQueue
 {
@@ -33,6 +34,19 @@ class ServerCheckJob implements ShouldBeEncrypted, ShouldQueue
 
     public function __construct(public Server $server) {}
 
+    public function failed(?\Throwable $exception): void
+    {
+        if ($exception instanceof \Illuminate\Queue\TimeoutExceededException) {
+            Log::warning('ServerCheckJob timed out', [
+                'server_id' => $this->server->id,
+                'server_name' => $this->server->name,
+            ]);
+
+            // Delete the queue job so it doesn't appear in Horizon's failed list.
+            $this->job?->delete();
+        }
+    }
+
     public function handle()
     {
         try {

app/Jobs/ServerConnectionCheckJob.php

@@ -101,12 +101,31 @@ class ServerConnectionCheckJob implements ShouldBeEncrypted, ShouldQueue
                 'is_usable' => false,
             ]);
 
-            throw $e;
+            return;
+        }
+    }
+
+    public function failed(?\Throwable $exception): void
+    {
+        if ($exception instanceof \Illuminate\Queue\TimeoutExceededException) {
+            Log::warning('ServerConnectionCheckJob timed out', [
+                'server_id' => $this->server->id,
+                'server_name' => $this->server->name,
+            ]);
+            $this->server->settings->update([
+                'is_reachable' => false,
+                'is_usable' => false,
+            ]);
+
+            // Delete the queue job so it doesn't appear in Horizon's failed list.
+            $this->job?->delete();
         }
     }
 
     private function checkHetznerStatus(): void
     {
+        $status = null;
+
         try {
             $hetznerService = new \App\Services\HetznerService($this->server->cloudProviderToken->token);
             $serverData = $hetznerService->getServer($this->server->hetzner_server_id);

app/Jobs/ServerStorageCheckJob.php

@@ -10,6 +10,7 @@ use Illuminate\Contracts\Queue\ShouldQueue;
 use Illuminate\Foundation\Bus\Dispatchable;
 use Illuminate\Queue\InteractsWithQueue;
 use Illuminate\Queue\SerializesModels;
+use Illuminate\Support\Facades\Log;
 use Illuminate\Support\Facades\RateLimiter;
 use Laravel\Horizon\Contracts\Silenced;
 
@@ -28,6 +29,19 @@ class ServerStorageCheckJob implements ShouldBeEncrypted, ShouldQueue, Silenced
 
     public function __construct(public Server $server, public int|string|null $percentage = null) {}
 
+    public function failed(?\Throwable $exception): void
+    {
+        if ($exception instanceof \Illuminate\Queue\TimeoutExceededException) {
+            Log::warning('ServerStorageCheckJob timed out', [
+                'server_id' => $this->server->id,
+                'server_name' => $this->server->name,
+            ]);
+
+            // Delete the queue job so it doesn't appear in Horizon's failed list.
+            $this->job?->delete();
+        }
+    }
+
     public function handle()
     {
         try {