Merge branch 'next' into fix/deploy-key-dedicated-path-race

app/Jobs/ApplicationDeploymentJob.php

@@ -220,6 +220,7 @@ class ApplicationDeploymentJob implements ShouldBeEncrypted, ShouldQueue
         $this->restart_only = $this->restart_only && $this->application->build_pack !== 'dockerimage' && $this->application->build_pack !== 'dockerfile';
         $this->only_this_server = $this->application_deployment_queue->only_this_server;
         $this->dockerImagePreviewTag = $this->application_deployment_queue->docker_registry_image_tag;
+        $this->validateDockerRegistryImageConfiguration();
 
         $this->git_type = data_get($this->application_deployment_queue, 'git_type');
 
@@ -1106,7 +1107,7 @@ class ApplicationDeploymentJob implements ShouldBeEncrypted, ShouldQueue
                     'hidden' => true,
                 ],
             );
-            if ($this->application->docker_registry_image_tag) {
+            if ($this->shouldPushDockerRegistryImageTag()) {
                 // Tag image with docker_registry_image_tag
                 $this->application_deployment_queue->addLogEntry("Tagging and pushing image with {$this->application->docker_registry_image_tag} tag.");
                 $this->execute_remote_command(
@@ -1130,6 +1131,30 @@ class ApplicationDeploymentJob implements ShouldBeEncrypted, ShouldQueue
         }
     }
 
+    private function shouldPushDockerRegistryImageTag(): bool
+    {
+        if (blank($this->application->docker_registry_image_tag)) {
+            return false;
+        }
+
+        return $this->pull_request_id === 0;
+    }
+
+    private function validateDockerRegistryImageConfiguration(): void
+    {
+        if (! ValidationPatterns::isValidDockerImageName($this->application->docker_registry_image_name)) {
+            throw new DeploymentException('Docker registry image name contains invalid characters.');
+        }
+
+        if (! ValidationPatterns::isValidDockerImageTag($this->application->docker_registry_image_tag)) {
+            throw new DeploymentException('Docker registry image tag contains invalid characters.');
+        }
+
+        if (! ValidationPatterns::isValidDockerImageTag($this->dockerImagePreviewTag)) {
+            throw new DeploymentException('Docker registry preview image tag contains invalid characters.');
+        }
+    }
+
     private function generate_image_names()
     {
         if ($this->application->dockerfile) {
@@ -1293,12 +1318,8 @@ class ApplicationDeploymentJob implements ShouldBeEncrypted, ShouldQueue
             $sorted_environment_variables_preview = $this->application->runtime_environment_variables_preview->sortBy('id');
         }
         if ($this->build_pack === 'dockercompose') {
-            $sorted_environment_variables = $sorted_environment_variables->filter(function ($env) {
-                return ! str($env->key)->startsWith('SERVICE_FQDN_') && ! str($env->key)->startsWith('SERVICE_URL_') && ! str($env->key)->startsWith('SERVICE_NAME_');
-            });
-            $sorted_environment_variables_preview = $sorted_environment_variables_preview->filter(function ($env) {
-                return ! str($env->key)->startsWith('SERVICE_FQDN_') && ! str($env->key)->startsWith('SERVICE_URL_') && ! str($env->key)->startsWith('SERVICE_NAME_');
-            });
+            $sorted_environment_variables = $sorted_environment_variables->reject(fn (EnvironmentVariable $env) => $this->isGeneratedDockerComposeEnvironmentVariable($env));
+            $sorted_environment_variables_preview = $sorted_environment_variables_preview->reject(fn (EnvironmentVariable $env) => $this->isGeneratedDockerComposeEnvironmentVariable($env));
         }
         $ports = $this->application->main_port();
         $coolify_envs = $this->generate_coolify_env_variables();
@@ -1451,6 +1472,15 @@ class ApplicationDeploymentJob implements ShouldBeEncrypted, ShouldQueue
         return $envs;
     }
 
+    private function isGeneratedDockerComposeEnvironmentVariable(EnvironmentVariable $environmentVariable): bool
+    {
+        $key = str($environmentVariable->key);
+
+        return $key->startsWith('SERVICE_FQDN_')
+            || $key->startsWith('SERVICE_URL_')
+            || $key->startsWith('SERVICE_NAME_');
+    }
+
     private function save_runtime_environment_variables()
     {
         // This method saves the .env file with ALL runtime variables
@@ -1666,11 +1696,9 @@ class ApplicationDeploymentJob implements ShouldBeEncrypted, ShouldQueue
                 ->orderBy($this->application->settings->is_env_sorting_enabled ? 'key' : 'id')
                 ->get();
 
-            // For Docker Compose, filter out SERVICE_FQDN and SERVICE_URL as we generate these
+            // For Docker Compose, filter out generated SERVICE_* variables as we generate these
             if ($this->build_pack === 'dockercompose') {
-                $sorted_environment_variables = $sorted_environment_variables->filter(function ($env) {
-                    return ! str($env->key)->startsWith('SERVICE_FQDN_') && ! str($env->key)->startsWith('SERVICE_URL_');
-                });
+                $sorted_environment_variables = $sorted_environment_variables->reject(fn (EnvironmentVariable $env) => $this->isGeneratedDockerComposeEnvironmentVariable($env));
             }
 
             foreach ($sorted_environment_variables as $env) {
@@ -1719,11 +1747,9 @@ class ApplicationDeploymentJob implements ShouldBeEncrypted, ShouldQueue
                 ->orderBy($this->application->settings->is_env_sorting_enabled ? 'key' : 'id')
                 ->get();
 
-            // For Docker Compose, filter out SERVICE_FQDN and SERVICE_URL as we generate these with PR-specific values
+            // For Docker Compose, filter out generated SERVICE_* variables as we generate these with PR-specific values
             if ($this->build_pack === 'dockercompose') {
-                $sorted_environment_variables = $sorted_environment_variables->filter(function ($env) {
-                    return ! str($env->key)->startsWith('SERVICE_FQDN_') && ! str($env->key)->startsWith('SERVICE_URL_');
-                });
+                $sorted_environment_variables = $sorted_environment_variables->reject(fn (EnvironmentVariable $env) => $this->isGeneratedDockerComposeEnvironmentVariable($env));
             }
 
             foreach ($sorted_environment_variables as $env) {
@@ -2103,21 +2129,23 @@ class ApplicationDeploymentJob implements ShouldBeEncrypted, ShouldQueue
         $helperImage = "{$helperImage}:".getHelperVersion();
         // Get user home directory
         $this->serverUserHomeDir = instant_remote_process(['echo $HOME'], $this->server);
+        instant_remote_process(["mkdir -p {$this->serverUserHomeDir}/.docker/buildx"], $this->server);
         $this->dockerConfigFileExists = instant_remote_process(["test -f {$this->serverUserHomeDir}/.docker/config.json && echo 'OK' || echo 'NOK'"], $this->server);
 
         $env_flags = $this->generate_docker_env_flags_for_secrets();
+        $buildxMetadataVolume = "-v {$this->serverUserHomeDir}/.docker/buildx:/root/.docker/buildx";
         if ($this->use_build_server) {
             if ($this->dockerConfigFileExists === 'NOK') {
                 throw new DeploymentException('Docker config file (~/.docker/config.json) not found on the build server. Please run "docker login" to login to the docker registry on the server.');
             }
-            $runCommand = "docker run -d --name {$this->deployment_uuid} {$env_flags} --rm -v {$this->serverUserHomeDir}/.docker/config.json:/root/.docker/config.json:ro -v /var/run/docker.sock:/var/run/docker.sock {$helperImage}";
+            $runCommand = "docker run -d --name {$this->deployment_uuid} {$env_flags} --rm -v {$this->serverUserHomeDir}/.docker/config.json:/root/.docker/config.json:ro {$buildxMetadataVolume} -v /var/run/docker.sock:/var/run/docker.sock {$helperImage}";
         } else {
             if ($this->dockerConfigFileExists === 'OK') {
                 $safeNetwork = escapeshellarg($this->destination->network);
-                $runCommand = "docker run -d --network {$safeNetwork} --name {$this->deployment_uuid} {$env_flags} --rm -v {$this->serverUserHomeDir}/.docker/config.json:/root/.docker/config.json:ro -v /var/run/docker.sock:/var/run/docker.sock {$helperImage}";
+                $runCommand = "docker run -d --network {$safeNetwork} --name {$this->deployment_uuid} {$env_flags} --rm -v {$this->serverUserHomeDir}/.docker/config.json:/root/.docker/config.json:ro {$buildxMetadataVolume} -v /var/run/docker.sock:/var/run/docker.sock {$helperImage}";
             } else {
                 $safeNetwork = escapeshellarg($this->destination->network);
-                $runCommand = "docker run -d --network {$safeNetwork} --name {$this->deployment_uuid} {$env_flags} --rm -v /var/run/docker.sock:/var/run/docker.sock {$helperImage}";
+                $runCommand = "docker run -d --network {$safeNetwork} --name {$this->deployment_uuid} {$env_flags} --rm {$buildxMetadataVolume} -v /var/run/docker.sock:/var/run/docker.sock {$helperImage}";
             }
         }
         if ($firstTry) {
@@ -2222,11 +2250,22 @@ class ApplicationDeploymentJob implements ShouldBeEncrypted, ShouldQueue
             }
         }
         if (isset($this->application->git_branch)) {
-            $this->coolify_variables .= "COOLIFY_BRANCH={$this->application->git_branch} ";
+            $this->coolify_variables .= 'COOLIFY_BRANCH='.escapeShellValue($this->application->git_branch).' ';
         }
         $this->coolify_variables .= "COOLIFY_RESOURCE_UUID={$this->application->uuid} ";
     }
 
+    private function gitLsRemoteCommand(string $lsRemoteRef, ?string $identityFile = null): string
+    {
+        $sshCommand = "ssh -o ConnectTimeout=30 -p {$this->customPort} -o Port={$this->customPort} -o LogLevel=ERROR -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null";
+
+        if ($identityFile !== null) {
+            $sshCommand .= " -i {$identityFile} -o IdentitiesOnly=yes";
+        }
+
+        return 'GIT_SSH_COMMAND="'.$sshCommand.'" git ls-remote '.escapeshellarg($this->fullRepoUrl).' '.escapeshellarg($lsRemoteRef);
+    }
+
     private function check_git_if_build_needed()
     {
         if (is_object($this->source) && $this->source->getMorphClass() === GithubApp::class && $this->source->is_public === false) {
@@ -2273,7 +2312,7 @@ class ApplicationDeploymentJob implements ShouldBeEncrypted, ShouldQueue
                     executeInDocker($this->deployment_uuid, "chmod 600 {$customSshKeyLocation}"),
                 ],
                 [
-                    executeInDocker($this->deployment_uuid, "GIT_SSH_COMMAND=\"ssh -o ConnectTimeout=30 -p {$this->customPort} -o Port={$this->customPort} -o LogLevel=ERROR -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i {$customSshKeyLocation} -o IdentitiesOnly=yes\" git ls-remote {$this->fullRepoUrl} {$lsRemoteRef}"),
+                    executeInDocker($this->deployment_uuid, $this->gitLsRemoteCommand($lsRemoteRef, $customSshKeyLocation)),
                     'hidden' => true,
                     'save' => 'git_commit_sha',
                 ]
@@ -2281,7 +2320,7 @@ class ApplicationDeploymentJob implements ShouldBeEncrypted, ShouldQueue
         } else {
             $this->execute_remote_command(
                 [
-                    executeInDocker($this->deployment_uuid, "GIT_SSH_COMMAND=\"ssh -o ConnectTimeout=30 -p {$this->customPort} -o Port={$this->customPort} -o LogLevel=ERROR -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null\" git ls-remote {$this->fullRepoUrl} {$lsRemoteRef}"),
+                    executeInDocker($this->deployment_uuid, $this->gitLsRemoteCommand($lsRemoteRef)),
                     'hidden' => true,
                     'save' => 'git_commit_sha',
                 ],
@@ -3020,6 +3059,10 @@ COPY ./nginx.conf /etc/nginx/conf.d/default.conf");
                 ->where('is_buildtime', true)
                 ->get();
 
+            if ($this->build_pack === 'dockercompose') {
+                $envs = $envs->reject(fn (EnvironmentVariable $env) => $this->isGeneratedDockerComposeEnvironmentVariable($env));
+            }
+
             foreach ($envs as $env) {
                 $resolvedValue = $env->getResolvedValueWithServer($this->mainServer);
                 if (! is_null($resolvedValue)) {
@@ -3032,6 +3075,10 @@ COPY ./nginx.conf /etc/nginx/conf.d/default.conf");
                 ->where('is_buildtime', true)
                 ->get();
 
+            if ($this->build_pack === 'dockercompose') {
+                $envs = $envs->reject(fn (EnvironmentVariable $env) => $this->isGeneratedDockerComposeEnvironmentVariable($env));
+            }
+
             foreach ($envs as $env) {
                 $resolvedValue = $env->getResolvedValueWithServer($this->mainServer);
                 if (! is_null($resolvedValue)) {

app/Jobs/CleanupStaleMultiplexedConnections.php

@@ -0,0 +1,228 @@
+<?php
+
+namespace App\Jobs;
+
+use App\Models\Server;
+use Carbon\Carbon;
+use Illuminate\Bus\Queueable;
+use Illuminate\Contracts\Queue\ShouldQueue;
+use Illuminate\Foundation\Bus\Dispatchable;
+use Illuminate\Queue\InteractsWithQueue;
+use Illuminate\Queue\SerializesModels;
+use Illuminate\Support\Facades\Log;
+use Illuminate\Support\Facades\Process;
+use Illuminate\Support\Facades\Storage;
+
+class CleanupStaleMultiplexedConnections implements ShouldQueue
+{
+    use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
+
+    public function handle()
+    {
+        $this->cleanupStaleConnections();
+        $this->cleanupNonExistentServerConnections();
+        $this->cleanupOrphanedSshProcesses();
+        $this->cleanupOrphanedCloudflaredProcesses();
+    }
+
+    /**
+     * Kill backgrounded ssh master processes that lost the ControlPath socket
+     * race. Such processes are not masters, so ControlPersist never reaps them
+     * and they leak memory until the container restarts. A legitimate master
+     * always owns its socket file; an orphan has none.
+     *
+     * Processes younger than the minimum age are skipped: a freshly forked
+     * master creates its socket a few milliseconds after starting, so a young
+     * process with no socket may simply be mid-establish rather than orphaned.
+     */
+    private function cleanupOrphanedSshProcesses(): void
+    {
+        $muxDir = storage_path('app/ssh/mux');
+        $minAge = (int) config('constants.ssh.mux_orphan_min_age');
+
+        foreach ($this->listProcesses() as $process) {
+            // Backgrounded ssh master: current `ssh -fN` or legacy `ssh -fNM`.
+            if (! preg_match('#(^|/)ssh -fN#', $process['args'])) {
+                continue;
+            }
+
+            // Only ever touch ssh processes pointing at Coolify's mux directory.
+            if (! preg_match('#ControlPath=('.preg_quote($muxDir, '#').'/\S+)#', $process['args'], $pathMatch)) {
+                continue;
+            }
+
+            if ($process['etimes'] >= $minAge && ! file_exists($pathMatch[1])) {
+                $this->reapOrphan('ssh', $process);
+            }
+        }
+    }
+
+    /**
+     * Kill orphaned `cloudflared access ssh` proxy processes. Each is spawned
+     * as the SSH ProxyCommand transport for a Cloudflare Tunnel server and must
+     * die with its parent ssh. When that ssh is killed or orphaned (e.g. a lost
+     * mux master), the cloudflared process can leak and accumulate. A legitimate
+     * proxy always has a live ssh parent; one without is safe to reap.
+     *
+     * Processes younger than the minimum age are skipped so a proxy whose parent
+     * ssh is still starting up, or a transient `ssh -O check` proxy mid-exit, is
+     * never mistaken for an orphan.
+     */
+    private function cleanupOrphanedCloudflaredProcesses(): void
+    {
+        $minAge = (int) config('constants.ssh.mux_orphan_min_age');
+        $processes = $this->listProcesses();
+
+        $sshPids = [];
+        foreach ($processes as $process) {
+            // The ssh binary itself, not `cloudflared access ssh` (space before ssh).
+            if (preg_match('#(^|/)ssh\s#', $process['args'])) {
+                $sshPids[$process['pid']] = true;
+            }
+        }
+
+        foreach ($processes as $process) {
+            // `cloudflared access ssh`, never the `cloudflared tunnel` daemon.
+            if (! str_contains($process['args'], 'cloudflared access ssh')) {
+                continue;
+            }
+
+            // Orphaned when no live ssh process is its parent.
+            if ($process['etimes'] >= $minAge && ! isset($sshPids[$process['ppid']])) {
+                $this->reapOrphan('cloudflared', $process);
+            }
+        }
+    }
+
+    /**
+     * Reap a detected orphan process. When orphan reaping is disabled (the
+     * default), the orphan is only logged — a dry-run mode that lets operators
+     * verify what would be killed before enabling it for real.
+     *
+     * @param  array{pid: string, ppid: string, etimes: int, args: string}  $process
+     */
+    private function reapOrphan(string $kind, array $process): void
+    {
+        if (! config('constants.ssh.mux_orphan_reap_enabled')) {
+            Log::info("Orphaned {$kind} process detected (dry-run, not killed)", [
+                'pid' => $process['pid'],
+                'etimes' => $process['etimes'],
+                'command' => $process['args'],
+            ]);
+
+            return;
+        }
+
+        Process::run('kill '.escapeshellarg($process['pid']));
+        Log::info("Killed orphaned {$kind} process", [
+            'pid' => $process['pid'],
+            'etimes' => $process['etimes'],
+            'command' => $process['args'],
+        ]);
+    }
+
+    /**
+     * Snapshot of running processes.
+     *
+     * @return list<array{pid: string, ppid: string, etimes: int, args: string}>
+     */
+    private function listProcesses(): array
+    {
+        $ps = Process::run('ps -ww -eo pid=,ppid=,etimes=,args=');
+        if ($ps->exitCode() !== 0) {
+            return [];
+        }
+
+        $processes = [];
+        foreach (explode("\n", trim($ps->output())) as $line) {
+            if (! preg_match('/^\s*(\d+)\s+(\d+)\s+(\d+)\s+(.*)$/', $line, $matches)) {
+                continue;
+            }
+            $processes[] = [
+                'pid' => $matches[1],
+                'ppid' => $matches[2],
+                'etimes' => (int) $matches[3],
+                'args' => $matches[4],
+            ];
+        }
+
+        return $processes;
+    }
+
+    private function cleanupStaleConnections()
+    {
+        $muxFiles = Storage::disk('ssh-mux')->files();
+
+        foreach ($muxFiles as $muxFile) {
+            $serverUuid = $this->extractServerUuidFromMuxFile($muxFile);
+            $server = Server::where('uuid', $serverUuid)->first();
+
+            if (! $server) {
+                $this->removeMultiplexFile($muxFile, 'server_not_found');
+
+                continue;
+            }
+
+            $muxSocket = "/var/www/html/storage/app/ssh/mux/{$muxFile}";
+            $checkCommand = "ssh -O check -o ControlPath={$muxSocket} {$server->user}@{$server->ip} 2>/dev/null";
+            $checkProcess = Process::run($checkCommand);
+
+            if ($checkProcess->exitCode() !== 0) {
+                $this->removeMultiplexFile($muxFile, 'connection_check_failed');
+            } else {
+                $muxContent = Storage::disk('ssh-mux')->get($muxFile);
+                $establishedAt = Carbon::parse(substr($muxContent, 37));
+                $expirationTime = $establishedAt->addSeconds(config('constants.ssh.mux_persist_time'));
+
+                if (Carbon::now()->isAfter($expirationTime)) {
+                    $this->removeMultiplexFile($muxFile, 'expired');
+                }
+            }
+        }
+    }
+
+    private function cleanupNonExistentServerConnections()
+    {
+        $muxFiles = Storage::disk('ssh-mux')->files();
+        $existingServerUuids = Server::pluck('uuid')->toArray();
+
+        foreach ($muxFiles as $muxFile) {
+            $serverUuid = $this->extractServerUuidFromMuxFile($muxFile);
+            if (! in_array($serverUuid, $existingServerUuids)) {
+                $this->removeMultiplexFile($muxFile, 'server_does_not_exist');
+            }
+        }
+    }
+
+    private function extractServerUuidFromMuxFile($muxFile)
+    {
+        return substr($muxFile, 4);
+    }
+
+    /**
+     * Close and delete a stale mux socket file. When orphan reaping is disabled
+     * (the default), the file is only logged — a dry-run mode that lets operators
+     * verify what would be removed before enabling it for real.
+     */
+    private function removeMultiplexFile(string $muxFile, string $reason): void
+    {
+        if (! config('constants.ssh.mux_orphan_reap_enabled')) {
+            Log::info('Stale mux file detected (dry-run, not removed)', [
+                'file' => $muxFile,
+                'reason' => $reason,
+            ]);
+
+            return;
+        }
+
+        $muxSocket = "/var/www/html/storage/app/ssh/mux/{$muxFile}";
+        $closeCommand = "ssh -O exit -o ControlPath={$muxSocket} localhost 2>/dev/null";
+        Process::run($closeCommand);
+        Storage::disk('ssh-mux')->delete($muxFile);
+
+        Log::info('Removed stale mux file', [
+            'file' => $muxFile,
+            'reason' => $reason,
+        ]);
+    }
+}

app/Jobs/DatabaseBackupJob.php

@@ -668,12 +668,14 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue
     private function upload_to_s3(): void
     {
         if (is_null($this->s3)) {
+            $previousS3StorageId = $this->backup->s3_storage_id;
+
             $this->backup->update([
                 'save_s3' => false,
                 's3_storage_id' => null,
             ]);
 
-            throw new \Exception('S3 storage configuration is missing or has been deleted (S3 storage ID: '.($this->backup->s3_storage_id ?? 'null').'). S3 backup has been disabled for this schedule.');
+            throw new \Exception('S3 storage configuration is missing or has been deleted (S3 storage ID: '.($previousS3StorageId ?? 'null').'). S3 backup has been disabled for this schedule.');
         }
 
         try {

app/Jobs/ProcessGithubPullRequestWebhook.php

@@ -39,6 +39,7 @@ class ProcessGithubPullRequestWebhook implements ShouldBeEncrypted, ShouldQueue
         public string $commitSha,
         public ?string $authorAssociation,
         public string $fullName,
+        public bool $isForkPullRequest = false,
     ) {
         $this->onQueue('high');
     }
@@ -92,7 +93,17 @@ class ProcessGithubPullRequestWebhook implements ShouldBeEncrypted, ShouldQueue
 
         // Check if PR deployments from public contributors are restricted
         if (! $application->settings->is_pr_deployments_public_enabled) {
-            $trustedAssociations = ['OWNER', 'MEMBER', 'COLLABORATOR', 'CONTRIBUTOR'];
+            // Fork PRs carry untrusted code from a repository outside our control.
+            // GitHub's author_association cannot be trusted to gate these (it grants
+            // CONTRIBUTOR to anyone who has merely opened an issue/PR before), so fork
+            // PRs are never deployed automatically when public previews are off.
+            if ($this->isForkPullRequest) {
+                return;
+            }
+
+            // Same-repo (non-fork) branch PRs require push access to the base repo,
+            // so only trusted associations are allowed to trigger a deployment.
+            $trustedAssociations = ['OWNER', 'MEMBER', 'COLLABORATOR'];
             if (! in_array($this->authorAssociation, $trustedAssociations)) {
                 return;
             }