Merge remote-tracking branch 'origin/next' into audit-policies

# Conflicts:
#	tests/Unit/Policies/GithubAppPolicyTest.php
#	tests/Unit/Policies/SharedEnvironmentVariablePolicyTest.php

app/Actions/Server/InstallDocker.php

@@ -30,12 +30,14 @@ class InstallDocker
             );
             $caCertPath = config('constants.coolify.base_config_path').'/ssl/';
 
+            $base64Cert = base64_encode($serverCert->ssl_certificate);
+
             $commands = collect([
                 "mkdir -p $caCertPath",
                 "chown -R 9999:root $caCertPath",
                 "chmod -R 700 $caCertPath",
                 "rm -rf $caCertPath/coolify-ca.crt",
-                "echo '{$serverCert->ssl_certificate}' > $caCertPath/coolify-ca.crt",
+                "echo '{$base64Cert}' | base64 -d | tee $caCertPath/coolify-ca.crt > /dev/null",
                 "chmod 644 $caCertPath/coolify-ca.crt",
             ]);
             remote_process($commands, $server);

app/Console/Kernel.php

@@ -40,7 +40,7 @@ class Kernel extends ConsoleKernel
         }
 
         // $this->scheduleInstance->job(new CleanupStaleMultiplexedConnections)->hourly();
-        $this->scheduleInstance->command('cleanup:redis')->weekly();
+        $this->scheduleInstance->command('cleanup:redis --clear-locks')->daily();
 
         if (isDev()) {
             // Instance Jobs

app/Http/Controllers/Api/ApplicationsController.php

@@ -1002,7 +1002,7 @@ class ApplicationsController extends Controller
         if ($return instanceof \Illuminate\Http\JsonResponse) {
             return $return;
         }
-        $allowedFields = ['project_uuid', 'environment_name', 'environment_uuid', 'server_uuid', 'destination_uuid', 'type', 'name', 'description', 'is_static', 'is_spa', 'is_auto_deploy_enabled', 'is_force_https_enabled', 'domains', 'git_repository', 'git_branch', 'git_commit_sha', 'private_key_uuid', 'docker_registry_image_name', 'docker_registry_image_tag', 'build_pack', 'install_command', 'build_command', 'start_command', 'ports_exposes', 'ports_mappings', 'custom_network_aliases', 'base_directory', 'publish_directory', 'health_check_enabled', 'health_check_path', 'health_check_port', 'health_check_host', 'health_check_method', 'health_check_return_code', 'health_check_scheme', 'health_check_response_text', 'health_check_interval', 'health_check_timeout', 'health_check_retries', 'health_check_start_period', 'limits_memory', 'limits_memory_swap', 'limits_memory_swappiness', 'limits_memory_reservation', 'limits_cpus', 'limits_cpuset', 'limits_cpu_shares', 'custom_labels', 'custom_docker_run_options', 'post_deployment_command', 'post_deployment_command_container', 'pre_deployment_command', 'pre_deployment_command_container',  'manual_webhook_secret_github', 'manual_webhook_secret_gitlab', 'manual_webhook_secret_bitbucket', 'manual_webhook_secret_gitea', 'redirect', 'github_app_uuid', 'instant_deploy', 'dockerfile', 'dockerfile_location', 'docker_compose_location', 'docker_compose_raw', 'docker_compose_custom_start_command', 'docker_compose_custom_build_command', 'docker_compose_domains', 'watch_paths', 'use_build_server', 'static_image', 'custom_nginx_configuration', 'is_http_basic_auth_enabled', 'http_basic_auth_username', 'http_basic_auth_password', 'connect_to_docker_network', 'force_domain_override', 'autogenerate_domain', 'is_container_label_escape_enabled'];
+        $allowedFields = ['project_uuid', 'environment_name', 'environment_uuid', 'server_uuid', 'destination_uuid', 'type', 'name', 'description', 'is_static', 'is_spa', 'is_auto_deploy_enabled', 'is_force_https_enabled', 'domains', 'git_repository', 'git_branch', 'git_commit_sha', 'private_key_uuid', 'docker_registry_image_name', 'docker_registry_image_tag', 'build_pack', 'install_command', 'build_command', 'start_command', 'ports_exposes', 'ports_mappings', 'custom_network_aliases', 'base_directory', 'publish_directory', 'health_check_enabled', 'health_check_type', 'health_check_command', 'health_check_path', 'health_check_port', 'health_check_host', 'health_check_method', 'health_check_return_code', 'health_check_scheme', 'health_check_response_text', 'health_check_interval', 'health_check_timeout', 'health_check_retries', 'health_check_start_period', 'limits_memory', 'limits_memory_swap', 'limits_memory_swappiness', 'limits_memory_reservation', 'limits_cpus', 'limits_cpuset', 'limits_cpu_shares', 'custom_labels', 'custom_docker_run_options', 'post_deployment_command', 'post_deployment_command_container', 'pre_deployment_command', 'pre_deployment_command_container',  'manual_webhook_secret_github', 'manual_webhook_secret_gitlab', 'manual_webhook_secret_bitbucket', 'manual_webhook_secret_gitea', 'redirect', 'github_app_uuid', 'instant_deploy', 'dockerfile', 'dockerfile_location', 'docker_compose_location', 'docker_compose_raw', 'docker_compose_custom_start_command', 'docker_compose_custom_build_command', 'docker_compose_domains', 'watch_paths', 'use_build_server', 'static_image', 'custom_nginx_configuration', 'is_http_basic_auth_enabled', 'http_basic_auth_username', 'http_basic_auth_password', 'connect_to_docker_network', 'force_domain_override', 'autogenerate_domain', 'is_container_label_escape_enabled'];
 
         $validator = customApiValidator($request->all(), [
             'name' => 'string|max:255',
@@ -2460,7 +2460,7 @@ class ApplicationsController extends Controller
         $this->authorize('update', $application);
 
         $server = $application->destination->server;
-        $allowedFields = ['name', 'description', 'is_static', 'is_spa', 'is_auto_deploy_enabled', 'is_force_https_enabled', 'domains', 'git_repository', 'git_branch', 'git_commit_sha', 'docker_registry_image_name', 'docker_registry_image_tag', 'build_pack', 'static_image', 'install_command', 'build_command', 'start_command', 'ports_exposes', 'ports_mappings', 'custom_network_aliases', 'base_directory', 'publish_directory', 'health_check_enabled', 'health_check_path', 'health_check_port', 'health_check_host', 'health_check_method', 'health_check_return_code', 'health_check_scheme', 'health_check_response_text', 'health_check_interval', 'health_check_timeout', 'health_check_retries', 'health_check_start_period', 'limits_memory', 'limits_memory_swap', 'limits_memory_swappiness', 'limits_memory_reservation', 'limits_cpus', 'limits_cpuset', 'limits_cpu_shares', 'custom_labels', 'custom_docker_run_options', 'post_deployment_command', 'post_deployment_command_container', 'pre_deployment_command', 'pre_deployment_command_container', 'watch_paths', 'manual_webhook_secret_github', 'manual_webhook_secret_gitlab', 'manual_webhook_secret_bitbucket', 'manual_webhook_secret_gitea', 'dockerfile_location', 'docker_compose_location', 'docker_compose_custom_start_command', 'docker_compose_custom_build_command', 'docker_compose_domains', 'redirect', 'instant_deploy', 'use_build_server', 'custom_nginx_configuration', 'is_http_basic_auth_enabled', 'http_basic_auth_username', 'http_basic_auth_password', 'connect_to_docker_network', 'force_domain_override', 'is_container_label_escape_enabled'];
+        $allowedFields = ['name', 'description', 'is_static', 'is_spa', 'is_auto_deploy_enabled', 'is_force_https_enabled', 'domains', 'git_repository', 'git_branch', 'git_commit_sha', 'docker_registry_image_name', 'docker_registry_image_tag', 'build_pack', 'static_image', 'install_command', 'build_command', 'start_command', 'ports_exposes', 'ports_mappings', 'custom_network_aliases', 'base_directory', 'publish_directory', 'health_check_enabled', 'health_check_type', 'health_check_command', 'health_check_path', 'health_check_port', 'health_check_host', 'health_check_method', 'health_check_return_code', 'health_check_scheme', 'health_check_response_text', 'health_check_interval', 'health_check_timeout', 'health_check_retries', 'health_check_start_period', 'limits_memory', 'limits_memory_swap', 'limits_memory_swappiness', 'limits_memory_reservation', 'limits_cpus', 'limits_cpuset', 'limits_cpu_shares', 'custom_labels', 'custom_docker_run_options', 'post_deployment_command', 'post_deployment_command_container', 'pre_deployment_command', 'pre_deployment_command_container', 'watch_paths', 'manual_webhook_secret_github', 'manual_webhook_secret_gitlab', 'manual_webhook_secret_bitbucket', 'manual_webhook_secret_gitea', 'dockerfile_location', 'docker_compose_location', 'docker_compose_custom_start_command', 'docker_compose_custom_build_command', 'docker_compose_domains', 'redirect', 'instant_deploy', 'use_build_server', 'custom_nginx_configuration', 'is_http_basic_auth_enabled', 'http_basic_auth_username', 'http_basic_auth_password', 'connect_to_docker_network', 'force_domain_override', 'is_container_label_escape_enabled'];
 
         $validationRules = [
             'name' => 'string|max:255',

app/Jobs/ApplicationDeploymentJob.php

@@ -1810,7 +1810,8 @@ class ApplicationDeploymentJob implements ShouldBeEncrypted, ShouldQueue
                     $counter = 1;
                     $this->application_deployment_queue->addLogEntry('Waiting for healthcheck to pass on the new container.');
                     if ($this->full_healthcheck_url && ! $this->application->custom_healthcheck_found) {
-                        $this->application_deployment_queue->addLogEntry("Healthcheck URL (inside the container): {$this->full_healthcheck_url}");
+                        $healthcheckLabel = $this->application->health_check_type === 'cmd' ? 'Healthcheck command' : 'Healthcheck URL';
+                        $this->application_deployment_queue->addLogEntry("{$healthcheckLabel} (inside the container): {$this->full_healthcheck_url}");
                     }
                     $this->application_deployment_queue->addLogEntry("Waiting for the start period ({$this->application->health_check_start_period} seconds) before starting healthcheck.");
                     $sleeptime = 0;
@@ -2768,6 +2769,14 @@ class ApplicationDeploymentJob implements ShouldBeEncrypted, ShouldQueue
 
     private function generate_healthcheck_commands()
     {
+        // Handle CMD type healthcheck
+        if ($this->application->health_check_type === 'cmd' && ! empty($this->application->health_check_command)) {
+            $this->full_healthcheck_url = $this->application->health_check_command;
+
+            return $this->application->health_check_command;
+        }
+
+        // HTTP type healthcheck (default)
         if (! $this->application->health_check_port) {
             $health_check_port = (int) $this->application->ports_exposes_array[0];
         } else {

app/Jobs/ScheduledJobManager.php

@@ -15,7 +15,9 @@ use Illuminate\Queue\Middleware\WithoutOverlapping;
 use Illuminate\Queue\SerializesModels;
 use Illuminate\Support\Carbon;
 use Illuminate\Support\Collection;
+use Illuminate\Support\Facades\Cache;
 use Illuminate\Support\Facades\Log;
+use Illuminate\Support\Facades\Redis;
 
 class ScheduledJobManager implements ShouldQueue
 {
@@ -54,6 +56,11 @@ class ScheduledJobManager implements ShouldQueue
      */
     public function middleware(): array
     {
+        // Self-healing: clear any stale lock before WithoutOverlapping tries to acquire it.
+        // Stale locks (TTL = -1) can occur during upgrades, Redis restarts, or edge cases.
+        // @see https://github.com/coollabsio/coolify/issues/8327
+        self::clearStaleLockIfPresent();
+
         return [
             (new WithoutOverlapping('scheduled-job-manager'))
                 ->expireAfter(90)   // Lock expires after 90s to handle high-load environments with many tasks
@@ -61,6 +68,34 @@ class ScheduledJobManager implements ShouldQueue
         ];
     }
 
+    /**
+     * Clear a stale WithoutOverlapping lock if it has no TTL (TTL = -1).
+     *
+     * This provides continuous self-healing since it runs every time the job is dispatched.
+     * Stale locks permanently block all scheduled job executions with no user-visible error.
+     */
+    private static function clearStaleLockIfPresent(): void
+    {
+        try {
+            $cachePrefix = config('cache.prefix', '');
+            $lockKey = $cachePrefix.'laravel-queue-overlap:'.self::class.':scheduled-job-manager';
+
+            $ttl = Redis::connection('default')->ttl($lockKey);
+
+            if ($ttl === -1) {
+                Redis::connection('default')->del($lockKey);
+                Log::channel('scheduled')->warning('Cleared stale ScheduledJobManager lock', [
+                    'lock_key' => $lockKey,
+                ]);
+            }
+        } catch (\Throwable $e) {
+            // Never let lock cleanup failure prevent the job from running
+            Log::channel('scheduled-errors')->error('Failed to check/clear stale lock', [
+                'error' => $e->getMessage(),
+            ]);
+        }
+    }
+
     public function handle(): void
     {
         // Freeze the execution time at the start of the job
@@ -108,6 +143,13 @@ class ScheduledJobManager implements ShouldQueue
             'dispatched' => $this->dispatchedCount,
             'skipped' => $this->skippedCount,
         ]);
+
+        // Write heartbeat so the UI can detect when the scheduler has stopped
+        try {
+            Cache::put('scheduled-job-manager:heartbeat', now()->toIso8601String(), 300);
+        } catch (\Throwable) {
+            // Non-critical; don't let heartbeat failure affect the job
+        }
     }
 
     private function processScheduledBackups(): void

app/Livewire/Project/Shared/HealthChecks.php

@@ -16,6 +16,12 @@ class HealthChecks extends Component
     #[Validate(['boolean'])]
     public bool $healthCheckEnabled = false;
 
+    #[Validate(['string', 'in:http,cmd'])]
+    public string $healthCheckType = 'http';
+
+    #[Validate(['nullable', 'required_if:healthCheckType,cmd', 'string', 'max:1000', 'regex:/^[a-zA-Z0-9 \-_.\/:=@,+]+$/'])]
+    public ?string $healthCheckCommand = null;
+
     #[Validate(['required', 'string', 'in:GET,HEAD,POST,OPTIONS'])]
     public string $healthCheckMethod;
 
@@ -54,6 +60,8 @@ class HealthChecks extends Component
 
     protected $rules = [
         'healthCheckEnabled' => 'boolean',
+        'healthCheckType' => 'string|in:http,cmd',
+        'healthCheckCommand' => ['nullable', 'string', 'max:1000', 'regex:/^[a-zA-Z0-9 \-_.\/:=@,+]+$/'],
         'healthCheckPath' => ['required', 'string', 'regex:#^[a-zA-Z0-9/\-_.~%]+$#'],
         'healthCheckPort' => 'nullable|integer|min:1|max:65535',
         'healthCheckHost' => ['required', 'string', 'regex:/^[a-zA-Z0-9.\-_]+$/'],
@@ -85,6 +93,8 @@ class HealthChecks extends Component
 
             // Sync to model
             $this->resource->health_check_enabled = $this->healthCheckEnabled;
+            $this->resource->health_check_type = $this->healthCheckType;
+            $this->resource->health_check_command = $this->healthCheckCommand;
             $this->resource->health_check_method = $this->healthCheckMethod;
             $this->resource->health_check_scheme = $this->healthCheckScheme;
             $this->resource->health_check_host = $this->healthCheckHost;
@@ -102,6 +112,8 @@ class HealthChecks extends Component
         } else {
             // Sync from model
             $this->healthCheckEnabled = $this->resource->health_check_enabled;
+            $this->healthCheckType = $this->resource->health_check_type ?? 'http';
+            $this->healthCheckCommand = $this->resource->health_check_command;
             $this->healthCheckMethod = $this->resource->health_check_method;
             $this->healthCheckScheme = $this->resource->health_check_scheme;
             $this->healthCheckHost = $this->resource->health_check_host;
@@ -120,9 +132,12 @@ class HealthChecks extends Component
     public function instantSave()
     {
         $this->authorize('update', $this->resource);
+        $this->validate();
 
         // Sync component properties to model
         $this->resource->health_check_enabled = $this->healthCheckEnabled;
+        $this->resource->health_check_type = $this->healthCheckType;
+        $this->resource->health_check_command = $this->healthCheckCommand;
         $this->resource->health_check_method = $this->healthCheckMethod;
         $this->resource->health_check_scheme = $this->healthCheckScheme;
         $this->resource->health_check_host = $this->healthCheckHost;
@@ -147,6 +162,8 @@ class HealthChecks extends Component
 
             // Sync component properties to model
             $this->resource->health_check_enabled = $this->healthCheckEnabled;
+            $this->resource->health_check_type = $this->healthCheckType;
+            $this->resource->health_check_command = $this->healthCheckCommand;
             $this->resource->health_check_method = $this->healthCheckMethod;
             $this->resource->health_check_scheme = $this->healthCheckScheme;
             $this->resource->health_check_host = $this->healthCheckHost;
@@ -175,6 +192,8 @@ class HealthChecks extends Component
 
             // Sync component properties to model
             $this->resource->health_check_enabled = $this->healthCheckEnabled;
+            $this->resource->health_check_type = $this->healthCheckType;
+            $this->resource->health_check_command = $this->healthCheckCommand;
             $this->resource->health_check_method = $this->healthCheckMethod;
             $this->resource->health_check_scheme = $this->healthCheckScheme;
             $this->resource->health_check_host = $this->healthCheckHost;

app/Livewire/Server/CaCertificate/Show.php

@@ -60,10 +60,16 @@ class Show extends Component
                 throw new \Exception('Certificate content cannot be empty.');
             }
 
-            if (! openssl_x509_read($this->certificateContent)) {
+            $parsedCert = openssl_x509_read($this->certificateContent);
+            if (! $parsedCert) {
                 throw new \Exception('Invalid certificate format.');
             }
 
+            if (! openssl_x509_export($parsedCert, $cleanedCertificate)) {
+                throw new \Exception('Failed to process certificate.');
+            }
+            $this->certificateContent = $cleanedCertificate;
+
             if ($this->caCertificate) {
                 $this->caCertificate->ssl_certificate = $this->certificateContent;
                 $this->caCertificate->save();
@@ -114,12 +120,14 @@ class Show extends Component
     {
         $caCertPath = config('constants.coolify.base_config_path').'/ssl/';
 
+        $base64Cert = base64_encode($this->certificateContent);
+
         $commands = collect([
             "mkdir -p $caCertPath",
             "chown -R 9999:root $caCertPath",
             "chmod -R 700 $caCertPath",
             "rm -rf $caCertPath/coolify-ca.crt",
-            "echo '{$this->certificateContent}' > $caCertPath/coolify-ca.crt",
+            "echo '{$base64Cert}' | base64 -d | tee $caCertPath/coolify-ca.crt > /dev/null",
             "chmod 644 $caCertPath/coolify-ca.crt",
         ]);
 

app/Livewire/Server/DockerCleanup.php

@@ -3,8 +3,13 @@
 namespace App\Livewire\Server;
 
 use App\Jobs\DockerCleanupJob;
+use App\Models\DockerCleanupExecution;
 use App\Models\Server;
+use Cron\CronExpression;
 use Illuminate\Foundation\Auth\Access\AuthorizesRequests;
+use Illuminate\Support\Carbon;
+use Illuminate\Support\Facades\Cache;
+use Livewire\Attributes\Computed;
 use Livewire\Attributes\Validate;
 use Livewire\Component;
 
@@ -34,6 +39,53 @@ class DockerCleanup extends Component
     #[Validate('boolean')]
     public bool $disableApplicationImageRetention = false;
 
+    #[Computed]
+    public function isCleanupStale(): bool
+    {
+        try {
+            $lastExecution = DockerCleanupExecution::where('server_id', $this->server->id)
+                ->orderBy('created_at', 'desc')
+                ->first();
+
+            if (! $lastExecution) {
+                return false;
+            }
+
+            $frequency = $this->server->settings->docker_cleanup_frequency ?? '0 0 * * *';
+            if (isset(VALID_CRON_STRINGS[$frequency])) {
+                $frequency = VALID_CRON_STRINGS[$frequency];
+            }
+
+            $cron = new CronExpression($frequency);
+            $now = Carbon::now();
+            $nextRun = Carbon::parse($cron->getNextRunDate($now));
+            $afterThat = Carbon::parse($cron->getNextRunDate($nextRun));
+            $intervalMinutes = $nextRun->diffInMinutes($afterThat);
+
+            $threshold = max($intervalMinutes * 2, 10);
+
+            return Carbon::parse($lastExecution->created_at)->diffInMinutes($now) > $threshold;
+        } catch (\Throwable) {
+            return false;
+        }
+    }
+
+    #[Computed]
+    public function lastExecutionTime(): ?string
+    {
+        return DockerCleanupExecution::where('server_id', $this->server->id)
+            ->orderBy('created_at', 'desc')
+            ->first()
+            ?->created_at
+            ?->diffForHumans();
+    }
+
+    #[Computed]
+    public function isSchedulerHealthy(): bool
+    {
+        return Cache::get('scheduled-job-manager:heartbeat') !== null;
+    }
+
     public function mount(string $server_uuid)
     {
         try {

app/Models/Application.php

@@ -61,6 +61,8 @@ use Visus\Cuid2\Cuid2;
         'health_check_timeout' => ['type' => 'integer', 'description' => 'Health check timeout in seconds.'],
         'health_check_retries' => ['type' => 'integer', 'description' => 'Health check retries count.'],
         'health_check_start_period' => ['type' => 'integer', 'description' => 'Health check start period in seconds.'],
+        'health_check_type' => ['type' => 'string', 'description' => 'Health check type: http or cmd.', 'enum' => ['http', 'cmd']],
+        'health_check_command' => ['type' => 'string', 'nullable' => true, 'description' => 'Health check command for CMD type.'],
         'limits_memory' => ['type' => 'string', 'description' => 'Memory limit.'],
         'limits_memory_swap' => ['type' => 'string', 'description' => 'Memory swap limit.'],
         'limits_memory_swappiness' => ['type' => 'integer', 'description' => 'Memory swappiness.'],
@@ -1959,7 +1961,6 @@ class Application extends BaseModel
         }
     }
 
-
     public function getLimits(): array
     {
         return [

app/Models/Server.php

@@ -1452,12 +1452,14 @@ $schema://$host {
                 $certificateContent = $caCertificate->ssl_certificate;
                 $caCertPath = config('constants.coolify.base_config_path').'/ssl/';
 
+                $base64Cert = base64_encode($certificateContent);
+
                 $commands = collect([
                     "mkdir -p $caCertPath",
                     "chown -R 9999:root $caCertPath",
                     "chmod -R 700 $caCertPath",
                     "rm -rf $caCertPath/coolify-ca.crt",
-                    "echo '{$certificateContent}' > $caCertPath/coolify-ca.crt",
+                    "echo '{$base64Cert}' | base64 -d | tee $caCertPath/coolify-ca.crt > /dev/null",
                     "chmod 644 $caCertPath/coolify-ca.crt",
                 ]);