fix(ssh): serialize initial mux connection creation

Wrap first-use SSH and SCP multiplexed commands with a lock to avoid racing while the control socket is created. Also detect native OpenSSH mux master process names during stale connection cleanup and cover both orphaned and duplicate mux processes with tests.

app/Helpers/SshMultiplexingHelper.php

@@ -41,13 +41,14 @@ class SshMultiplexingHelper
     {
         $sshConfig = self::serverSshConfiguration($server);
         $sshKeyLocation = $sshConfig['sshKeyLocation'];
+        $multiplexingEnabled = self::isMultiplexingEnabled();
         $scpCommand = 'timeout '.config('constants.ssh.command_timeout').' scp ';
 
         if ($server->isIpv6()) {
             $scpCommand .= '-6 ';
         }
 
-        if (self::isMultiplexingEnabled()) {
+        if ($multiplexingEnabled) {
             $scpCommand .= self::multiplexingOptions($server);
         }
 
@@ -58,10 +59,14 @@ class SshMultiplexingHelper
         $scpCommand .= self::getCommonSshOptions($server, $sshKeyLocation, self::getConnectionTimeout($server), config('constants.ssh.server_interval'), isScp: true);
 
         if ($server->isIpv6()) {
-            return $scpCommand."{$source} ".escapeshellarg($server->user).'@['.escapeshellarg($server->ip)."]:{$dest}";
+            $scpCommand .= "{$source} ".escapeshellarg($server->user).'@['.escapeshellarg($server->ip)."]:{$dest}";
+        } else {
+            $scpCommand .= "{$source} ".self::escapedUserAtHost($server).":{$dest}";
         }
 
-        return $scpCommand."{$source} ".self::escapedUserAtHost($server).":{$dest}";
+        return $multiplexingEnabled
+            ? self::withFirstUseMuxLock($server, $scpCommand)
+            : $scpCommand;
     }
 
     public static function generateSshCommand(Server $server, string $command, bool $disableMultiplexing = false): string
@@ -72,12 +77,13 @@ class SshMultiplexingHelper
 
         $sshConfig = self::serverSshConfiguration($server);
         $sshKeyLocation = $sshConfig['sshKeyLocation'];
+        $multiplexingEnabled = ! $disableMultiplexing && self::isMultiplexingEnabled();
 
         self::validateSshKey($server->privateKey);
 
         $sshCommand = 'timeout '.config('constants.ssh.command_timeout').' ssh ';
 
-        if (! $disableMultiplexing && self::isMultiplexingEnabled()) {
+        if ($multiplexingEnabled) {
             $sshCommand .= self::multiplexingOptions($server);
         }
 
@@ -90,9 +96,13 @@ class SshMultiplexingHelper
         $delimiter = base64_encode(Hash::make($command));
         $command = str_replace($delimiter, '', $command);
 
-        return $sshCommand.self::escapedUserAtHost($server)." 'bash -se' << \\$delimiter".PHP_EOL
+        $sshCommand .= self::escapedUserAtHost($server)." 'bash -se' << \\$delimiter".PHP_EOL
             .$command.PHP_EOL
             .$delimiter;
+
+        return $multiplexingEnabled
+            ? self::withFirstUseMuxLock($server, $sshCommand)
+            : $sshCommand;
     }
 
     private static function multiplexingOptions(Server $server): string
@@ -107,6 +117,78 @@ class SshMultiplexingHelper
         return '/var/www/html/storage/app/ssh/mux/mux_'.$server->uuid;
     }
 
+    private static function muxLockDirectory(Server $server): string
+    {
+        return self::muxSocket($server).'.lock';
+    }
+
+    private static function withFirstUseMuxLock(Server $server, string $command): string
+    {
+        $muxSocket = self::muxSocket($server);
+        $lockDirectory = self::muxLockDirectory($server);
+        $lockTimeout = (int) config('constants.ssh.mux_lock_timeout');
+
+        $script = <<<'SH'
+cmd=$1
+socket=$2
+lock=$3
+timeout=$4
+
+run_command() {
+    sh -c "$cmd"
+}
+
+if [ -S "$socket" ]; then
+    run_command
+    exit $?
+fi
+
+waited=0
+while ! mkdir "$lock" 2>/dev/null; do
+    if [ -S "$socket" ]; then
+        run_command
+        exit $?
+    fi
+
+    if [ "$waited" -ge "$timeout" ]; then
+        run_command
+        exit $?
+    fi
+
+    waited=$((waited + 1))
+    sleep 1
+done
+
+cleanup() {
+    if [ -n "${child:-}" ] && kill -0 "$child" 2>/dev/null; then
+        kill "$child" 2>/dev/null
+    fi
+    rmdir "$lock" 2>/dev/null
+}
+trap cleanup INT TERM HUP
+
+sh -c "$cmd" &
+child=$!
+
+for _ in 1 2 3 4 5 6 7 8 9 10; do
+    if [ -S "$socket" ] || ! kill -0 "$child" 2>/dev/null; then
+        break
+    fi
+    sleep 0.1
+done
+
+rmdir "$lock" 2>/dev/null
+wait "$child"
+exit $?
+SH;
+
+        return 'sh -c '.escapeshellarg($script).' -- '
+            .escapeshellarg($command).' '
+            .escapeshellarg($muxSocket).' '
+            .escapeshellarg($lockDirectory).' '
+            .escapeshellarg((string) $lockTimeout);
+    }
+
     private static function escapedUserAtHost(Server $server): string
     {
         return escapeshellarg($server->user).'@'.escapeshellarg($server->ip);