feat(v5): add server reconciliation and canvas APIs

Split V5 dashboard behavior into domain controllers and policies,
add agent token rotation/revocation, status reconciliation jobs,
ingress firewall syncing, and canvas connection APIs.

Add migrations for V5 status tracking, server capabilities, resource
connection aliases, and revoked agent tokens.

app/Actions/V5/Application/DeployNginxApplication.php

@@ -2,8 +2,12 @@
 
 namespace App\Actions\V5\Application;
 
+use App\Enums\V5\ApplicationStatus;
+use App\Enums\V5\ContainerState;
+use App\Enums\V5\ServerStatus;
 use App\Models\V5\Application;
 use App\Services\Flux\FluxClient;
+use Illuminate\Support\Facades\Log;
 use Lorisleiva\Actions\Concerns\AsAction;
 
 class DeployNginxApplication
@@ -21,34 +25,91 @@ class DeployNginxApplication
             return $this->markFailed($application, 'No server is attached to this application.');
         }
 
-        $hostId = $server->wireguard_management_ip ?: $server->node_address ?: $server->host;
+        if ($server->status !== ServerStatus::Installed->value || $server->last_bootstrapped_at === null) {
+            return $this->markFailed($application, "Bootstrap server {$server->name} before deploying to it.");
+        }
+
+        $hostId = $server->fluxHostId();
 
         if (! is_string($hostId) || $hostId === '') {
             return $this->markFailed($application, 'No Flux host ID is available for this server.');
         }
 
+        $containerId = null;
+
         try {
             $this->fluxClient->pullImage($hostId, $application->image);
             $containerId = $this->fluxClient->createContainer($hostId, $this->containerSpec($application));
+
+            // Persist the runtime id the instant the container exists, before
+            // start/inspect. A worker SIGKILL at the job timeout would otherwise
+            // orphan a created container whose id only lived in this local var,
+            // leaving failed()/reconcile unable to find and clean it by id.
+            $application->update([
+                'status' => ApplicationStatus::Created->value,
+                'status_message' => 'Container created.',
+                'runtime_container_id' => $containerId,
+            ]);
+
             $this->fluxClient->startContainer($hostId, $containerId);
             $inspect = $this->fluxClient->inspectContainer($hostId, $containerId);
 
             if (! $this->isContainerRunning($inspect)) {
+                $this->cleanUpContainer($application, $hostId, $containerId);
+
                 return $this->markFailed($application, 'Container did not stay running.');
             }
 
             $application->update([
-                'status' => 'running',
+                'status' => ApplicationStatus::Running->value,
                 'status_message' => 'Container started.',
                 'runtime_container_id' => $containerId,
             ]);
 
             return $application->refresh()->load('server');
         } catch (\Throwable $e) {
+            if (is_string($containerId) && $containerId !== '') {
+                $this->cleanUpContainer($application, $hostId, $containerId);
+            }
+
             return $this->markFailed($application, $e->getMessage());
         }
     }
 
+    /**
+     * Best-effort compensation for a failed deploy: stop and force-remove the
+     * container this run created so it is never left orphaned on the node, then
+     * null the runtime id we persisted right after create so a cleaned-up
+     * failure never leaves a dangling id that reconcile would try to reap.
+     * Cleanup failures only log a warning and never mask the original error.
+     */
+    private function cleanUpContainer(Application $application, string $hostId, string $containerId): void
+    {
+        try {
+            $this->fluxClient->stopContainer($hostId, $containerId);
+        } catch (\Throwable $e) {
+            Log::warning('Could not stop the container created by a failed v5 deploy.', [
+                'application_id' => $application->getKey(),
+                'container_id' => $containerId,
+                'error' => $e->getMessage(),
+            ]);
+        }
+
+        try {
+            $this->fluxClient->removeContainer($hostId, $containerId, force: true);
+        } catch (\Throwable $e) {
+            Log::warning('Could not remove the container created by a failed v5 deploy.', [
+                'application_id' => $application->getKey(),
+                'container_id' => $containerId,
+                'error' => $e->getMessage(),
+            ]);
+        }
+
+        if ($application->runtime_container_id === $containerId) {
+            $application->update(['runtime_container_id' => null]);
+        }
+    }
+
     /**
      * @return array<string, mixed>
      */
@@ -92,13 +153,13 @@ class DeployNginxApplication
             return true;
         }
 
-        return is_string($inspect['state'] ?? null) && $inspect['state'] === 'running';
+        return is_string($inspect['state'] ?? null) && $inspect['state'] === ContainerState::Running->value;
     }
 
     private function markFailed(Application $application, string $message): Application
     {
         $application->update([
-            'status' => 'failed',
+            'status' => ApplicationStatus::Failed->value,
             'status_message' => str($message)->limit(10000)->toString(),
         ]);
 

app/Actions/V5/Flux/ApplyFluxResourceStatusUpdate.php

@@ -2,10 +2,18 @@
 
 namespace App\Actions\V5\Flux;
 
+use App\Enums\V5\ApplicationStatus;
+use App\Enums\V5\ContainerState;
+use App\Enums\V5\IngressStatus;
+use App\Enums\V5\ServerStatus;
 use App\Models\V5\Application as V5Application;
 use App\Models\V5\ContainerStatus;
 use App\Models\V5\Server as V5Server;
+use App\Support\V5\StatusObservation;
+use Carbon\CarbonImmutable;
+use Carbon\CarbonInterface;
 use Illuminate\Database\Eloquent\Model;
+use Illuminate\Support\Facades\Log;
 use Lorisleiva\Actions\Concerns\AsAction;
 
 class ApplyFluxResourceStatusUpdate
@@ -37,7 +45,7 @@ class ApplyFluxResourceStatusUpdate
      */
     private function upsertContainerStatus(array $payload): ?ContainerStatus
     {
-        $status = $this->status($payload);
+        $status = $this->status($payload, ContainerState::class);
         $containerId = $this->stringValue($payload, 'container_id') ?? $this->stringValue($payload, 'runtime_container_id');
         $server = $this->findServer($payload);
 
@@ -45,17 +53,36 @@ class ApplyFluxResourceStatusUpdate
             return null;
         }
 
-        ContainerStatus::query()->updateOrCreate([
+        $observedAt = $this->observedAt($payload);
+        $existing = ContainerStatus::query()
+            ->where('server_id', $server->id)
+            ->where('container_id', $containerId)
+            ->first();
+
+        if ($this->isStaleObservation($observedAt, $existing?->status_observed_at, 'container status', [
             'server_id' => $server->id,
             'container_id' => $containerId,
-        ], [
+        ])) {
+            return $existing;
+        }
+
+        $attributes = [
             'team_id' => $server->team_id,
             'container_name' => $this->stringValue($payload, 'container_name') ?? $this->stringValue($payload, 'name'),
             'image' => $this->stringValue($payload, 'image'),
             'status' => $status,
             'status_message' => $this->statusMessage($payload, 'Container state received from coold.'),
             'last_seen_at' => now(),
-        ]);
+        ];
+
+        if ($observedAt !== null) {
+            $attributes['status_observed_at'] = $observedAt;
+        }
+
+        ContainerStatus::query()->updateOrCreate([
+            'server_id' => $server->id,
+            'container_id' => $containerId,
+        ], $attributes);
 
         return ContainerStatus::query()
             ->where('server_id', $server->id)
@@ -68,7 +95,7 @@ class ApplyFluxResourceStatusUpdate
      */
     private function updateApplication(array $payload): ?V5Application
     {
-        $status = $this->status($payload);
+        $status = $this->status($payload, ApplicationStatus::class);
 
         if ($status === null) {
             return null;
@@ -80,13 +107,39 @@ class ApplyFluxResourceStatusUpdate
             return null;
         }
 
-        $application->update([
+        $observedAt = $this->observedAt($payload);
+
+        if ($this->isStaleObservation($observedAt, $application->status_observed_at, 'application status', [
+            'application_id' => $application->id,
+        ])) {
+            return $application;
+        }
+
+        $payloadContainerId = $this->stringValue($payload, 'runtime_container_id')
+            ?? $this->stringValue($payload, 'container_id');
+
+        // Payloads may carry no timestamp, so the container id remains an
+        // ordering signal as a second layer: an update for a superseded
+        // container is stale and must not overwrite the current one's state.
+        if (
+            $payloadContainerId !== null
+            && $application->runtime_container_id !== null
+            && $payloadContainerId !== $application->runtime_container_id
+        ) {
+            return $application;
+        }
+
+        $attributes = [
             'status' => $status,
             'status_message' => $this->statusMessage($payload, 'Status updated by flux.'),
-            'runtime_container_id' => $this->stringValue($payload, 'runtime_container_id')
-                ?? $this->stringValue($payload, 'container_id')
-                ?? $application->runtime_container_id,
-        ]);
+            'runtime_container_id' => $payloadContainerId ?? $application->runtime_container_id,
+        ];
+
+        if ($observedAt !== null) {
+            $attributes['status_observed_at'] = $observedAt;
+        }
+
+        $application->update($attributes);
 
         return $application->refresh();
     }
@@ -96,7 +149,7 @@ class ApplyFluxResourceStatusUpdate
      */
     private function updateServer(array $payload): ?V5Server
     {
-        $status = $this->status($payload);
+        $status = $this->status($payload, ServerStatus::class);
 
         if ($status === null) {
             return null;
@@ -108,22 +161,40 @@ class ApplyFluxResourceStatusUpdate
             return null;
         }
 
-        $server->update([
+        $observedAt = $this->observedAt($payload);
+
+        if ($this->isStaleObservation($observedAt, $server->status_observed_at, 'server status', [
+            'server_id' => $server->id,
+        ])) {
+            return $server;
+        }
+
+        $attributes = [
             'status' => $status,
             'last_status_check' => 'flux',
             'last_status_output' => $this->statusMessage($payload, 'Status updated by flux.'),
             'last_status_checked_at' => now(),
-        ]);
+        ];
+
+        if ($observedAt !== null) {
+            $attributes['status_observed_at'] = $observedAt;
+        }
+
+        $server->update($attributes);
 
         return $server->refresh();
     }
 
     /**
+     * The ingress state shares the server row but describes a different
+     * resource, so it deliberately does not read or write the server's
+     * `status_observed_at` watermark.
+     *
      * @param  array<string, mixed>  $payload
      */
     private function updateCaddyIngress(array $payload): ?V5Server
     {
-        $status = $this->status($payload);
+        $status = $this->status($payload, IngressStatus::class);
 
         if ($status === null) {
             return null;
@@ -151,13 +222,17 @@ class ApplyFluxResourceStatusUpdate
      */
     private function findApplication(array $payload): ?V5Application
     {
-        $query = V5Application::query()->with('server');
         $server = $this->findServer($payload);
 
-        if ($server instanceof V5Server) {
-            $query->where('server_id', $server->id);
+        if (! $server instanceof V5Server) {
+            return null;
         }
 
+        $query = V5Application::query()
+            ->with('server')
+            ->where('server_id', $server->id)
+            ->where('team_id', $server->team_id);
+
         $applicationUuid = $this->stringValue($payload, 'application_uuid') ?? $this->stringValue($payload, 'resource_uuid');
 
         if ($applicationUuid !== null) {
@@ -211,21 +286,62 @@ class ApplyFluxResourceStatusUpdate
             return null;
         }
 
-        return V5Server::query()
-            ->where('wireguard_management_ip', $hostId)
-            ->orWhere('node_address', $hostId)
-            ->orWhere('host', $hostId)
-            ->first();
+        $matches = V5Server::query()
+            ->where('uuid', $hostId)
+            ->limit(2)
+            ->get();
+
+        if ($matches->count() > 1) {
+            Log::warning('Dropping flux resource status update: host id matches multiple v5 servers.', [
+                'host_id' => $hostId,
+                'server_ids' => $matches->pluck('id')->all(),
+            ]);
+
+            return null;
+        }
+
+        return $matches->first();
+    }
+
+    /**
+     * Map the raw payload status onto the given status enum. Unknown values
+     * are never written to the database: they fall back to the enum's
+     * Unknown case and are logged.
+     *
+     * @param  array<string, mixed>  $payload
+     * @param  class-string<ApplicationStatus|ContainerState|IngressStatus|ServerStatus>  $enumClass
+     */
+    private function status(array $payload, string $enumClass): ?string
+    {
+        $raw = $this->stringValue($payload, 'status') ?? $this->stringValue($payload, 'state');
+
+        return StatusObservation::normalize($raw, $enumClass);
     }
 
     /**
      * @param  array<string, mixed>  $payload
      */
-    private function status(array $payload): ?string
+    private function observedAt(array $payload): ?CarbonInterface
     {
-        $status = $this->stringValue($payload, 'status') ?? $this->stringValue($payload, 'state');
+        $observedAt = $this->stringValue($payload, 'observed_at');
 
-        return $status === null ? null : strtolower($status);
+        if ($observedAt === null) {
+            return null;
+        }
+
+        return rescue(fn (): CarbonImmutable => CarbonImmutable::parse($observedAt), null, false);
+    }
+
+    /**
+     * A payload that carries an observation timestamp older than the one
+     * already persisted is stale (delivered out of order) and must not
+     * clobber the newer state.
+     *
+     * @param  array<string, mixed>  $logContext
+     */
+    private function isStaleObservation(?CarbonInterface $observedAt, ?CarbonInterface $currentObservedAt, string $context, array $logContext): bool
+    {
+        return StatusObservation::isStale($observedAt, $currentObservedAt, $context, $logContext);
     }
 
     /**
@@ -247,18 +363,4 @@ class ApplyFluxResourceStatusUpdate
 
         return is_string($value) && $value !== '' ? $value : null;
     }
-
-    /**
-     * @param  array<string, mixed>  $payload
-     */
-    private function intValue(array $payload, string $key): ?int
-    {
-        $value = data_get($payload, $key);
-
-        if (is_int($value)) {
-            return $value;
-        }
-
-        return is_string($value) && ctype_digit($value) ? (int) $value : null;
-    }
 }

app/Actions/V5/Proxy/GenerateCaddyIngressConfiguration.php

@@ -5,6 +5,7 @@ namespace App\Actions\V5\Proxy;
 use App\Models\V5\Application;
 use App\Models\V5\ApplicationDomain;
 use Illuminate\Support\Collection;
+use Illuminate\Support\Facades\Log;
 use Lorisleiva\Actions\Concerns\AsAction;
 use Symfony\Component\Yaml\Yaml;
 
@@ -12,6 +13,14 @@ class GenerateCaddyIngressConfiguration
 {
     use AsAction;
 
+    /**
+     * Strict RFC 1123 hostname: dot-separated alphanumeric labels with inner
+     * hyphens, max 253 characters. Anchored with \A/\z (never $) so values
+     * containing newlines, braces, quotes, whitespace, or control characters
+     * can never inject extra directives into the generated Caddyfile.
+     */
+    private const HOSTNAME_PATTERN = '/\A(?=.{1,253}\z)[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?(?:\.[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?)*\z/i';
+
     /**
      * @param  Collection<int, Application>|null  $applications
      * @return array{compose: string, caddyfile: string, apps: array<int, array{name: string, caddyfile: string}>}
@@ -92,12 +101,42 @@ CADDY;
 
     private function applicationRoute(Application $application, ApplicationDomain $domain): ?string
     {
-        if ($domain->domain === '') {
+        if ($domain->domain === null || $domain->domain === '') {
             return null;
         }
 
         $namespace = $application->mesh_namespace ?: 'default';
-        $upstream = "{$application->container_name}.{$namespace}.coolify.internal:{$application->internal_port}";
+        $internalPort = (int) $application->internal_port;
+
+        if (! $this->isSafeHostname($domain->domain)) {
+            Log::warning('Skipping a caddy ingress route with an unsafe domain.', [
+                'application_id' => $application->getKey(),
+                'domain' => $domain->domain,
+            ]);
+
+            return null;
+        }
+
+        if (! $this->isSafeHostname($application->container_name) || ! $this->isSafeHostname($namespace)) {
+            Log::warning('Skipping a caddy ingress route with an unsafe container name or namespace.', [
+                'application_id' => $application->getKey(),
+                'container_name' => $application->container_name,
+                'namespace' => $namespace,
+            ]);
+
+            return null;
+        }
+
+        if ($internalPort < 1 || $internalPort > 65535) {
+            Log::warning('Skipping a caddy ingress route with an out-of-range internal port.', [
+                'application_id' => $application->getKey(),
+                'internal_port' => $application->internal_port,
+            ]);
+
+            return null;
+        }
+
+        $upstream = "{$application->container_name}.{$namespace}.coolify.internal:{$internalPort}";
 
         return implode("\n", [
             "http://{$domain->domain} {",
@@ -106,6 +145,11 @@ CADDY;
         ]);
     }
 
+    private function isSafeHostname(mixed $value): bool
+    {
+        return is_string($value) && preg_match(self::HOSTNAME_PATTERN, $value) === 1;
+    }
+
     private function appFileName(Application $application): string
     {
         return 'app_'.$application->getKey();

app/Actions/V5/Proxy/StartCaddyIngress.php

@@ -2,17 +2,19 @@
 
 namespace App\Actions\V5\Proxy;
 
+use App\Exceptions\V5\UnsupportedCooldVerb;
 use App\Models\V5\Application;
 use App\Models\V5\Server;
 use App\Services\Flux\FluxClient;
 use Illuminate\Support\Collection;
+use Illuminate\Support\Facades\Log;
 use Lorisleiva\Actions\Concerns\AsAction;
 
 class StartCaddyIngress
 {
     use AsAction;
 
-    private const FIREWALL_PORTS = [80, 443];
+    private const FIREWALL_PORTS = [80];
 
     public function __construct(private readonly FluxClient $fluxClient) {}
 
@@ -22,30 +24,48 @@ class StartCaddyIngress
             return 'Server is not an ingress server.';
         }
 
-        $hostId = $server->wireguard_management_ip ?: $server->node_address;
+        $hostId = $server->fluxHostId();
 
         if (! is_string($hostId) || $hostId === '') {
-            return 'Server is missing its Flux host id.';
+            throw new \RuntimeException('Server is missing its Flux host id.');
         }
 
         $configuration = GenerateCaddyIngressConfiguration::run($this->applications($server));
         $output = $this->fluxClient->applyIngress($hostId, 'caddy', $configuration['caddyfile'], $this->ingressApps($configuration['apps']));
 
+        $firewallWarning = null;
+
         foreach (self::FIREWALL_PORTS as $port) {
-            $this->fluxClient->applyFirewallRule($hostId, [
-                'id' => "v5-caddy-ingress:{$port}",
-                'namespace' => 'default',
-                'src' => '0.0.0.0/0',
-                'dst' => 'coolify-v5-caddy',
-                'proto' => 'tcp',
-                'port' => $port,
-            ]);
+            try {
+                $this->fluxClient->applyFirewallRule($hostId, [
+                    'id' => "v5-caddy-ingress:{$port}",
+                    'namespace' => 'default',
+                    'src' => '0.0.0.0/0',
+                    'dst' => 'coolify-v5-caddy',
+                    'proto' => 'tcp',
+                    'port' => $port,
+                ]);
+            } catch (UnsupportedCooldVerb $exception) {
+                $firewallWarning = "Caddy ingress is running, but this node's coold does not support {$exception->verb}, so the managed firewall was not updated for port {$port}.";
+                Log::warning('V5 caddy ingress firewall rule skipped: coold verb unsupported', [
+                    'server_id' => $server->id,
+                    'port' => $port,
+                    'verb' => $exception->verb,
+                    'message' => $exception->getMessage(),
+                ]);
+
+                break;
+            }
         }
 
         if ($server->exists) {
             $server->update([
                 'ingress_type' => 'caddy',
                 'ingress_status' => 'running',
+                ...($firewallWarning === null ? [] : [
+                    'last_status_check' => 'flux',
+                    'last_status_output' => $firewallWarning,
+                ]),
             ]);
         }
 

app/Actions/V5/Proxy/StopCaddyIngress.php

@@ -2,36 +2,64 @@
 
 namespace App\Actions\V5\Proxy;
 
+use App\Enums\V5\IngressStatus;
+use App\Exceptions\V5\UnsupportedCooldVerb;
 use App\Models\V5\Server;
 use App\Services\Flux\FluxClient;
+use Illuminate\Support\Facades\Log;
+use Illuminate\Support\Str;
 use Lorisleiva\Actions\Concerns\AsAction;
 
 class StopCaddyIngress
 {
     use AsAction;
 
-    private const FIREWALL_PORTS = [80, 443];
+    private const FIREWALL_PORTS = [80];
 
     public function __construct(private readonly FluxClient $fluxClient) {}
 
     public function handle(Server $server): string
     {
-        $hostId = $server->wireguard_management_ip ?: $server->node_address;
+        if (! $server->isIngress() && $server->ingress_type === null) {
+            return 'Server is not an ingress server.';
+        }
+
+        $hostId = $server->fluxHostId();
 
         if (! is_string($hostId) || $hostId === '') {
-            return 'Server is missing its Flux host id.';
+            throw new \RuntimeException('Server is missing its Flux host id.');
+        }
+
+        // Revoke first: if stopping the container fails the allow rules must not
+        // stay orphaned on the host.
+        foreach (self::FIREWALL_PORTS as $port) {
+            $this->revokeFirewallRuleIfPresent($hostId, "v5-caddy-ingress:{$port}");
         }
 
         $output = $this->fluxClient->stopIngress($hostId, 'caddy');
 
-        foreach (self::FIREWALL_PORTS as $port) {
-            $this->fluxClient->revokeFirewallRule($hostId, "v5-caddy-ingress:{$port}");
-        }
-
         if ($server->exists) {
-            $server->update(['ingress_status' => 'exited']);
+            $server->update(['ingress_status' => IngressStatus::Exited->value]);
         }
 
         return $output;
     }
+
+    private function revokeFirewallRuleIfPresent(string $hostId, string $ruleId): void
+    {
+        try {
+            $this->fluxClient->revokeFirewallRule($hostId, $ruleId);
+        } catch (UnsupportedCooldVerb $exception) {
+            Log::warning('V5 caddy ingress firewall revoke skipped: coold verb unsupported', [
+                'host_id' => $hostId,
+                'rule_id' => $ruleId,
+                'verb' => $exception->verb,
+                'message' => $exception->getMessage(),
+            ]);
+        } catch (\RuntimeException $exception) {
+            if (! str_contains(Str::lower($exception->getMessage()), 'not found')) {
+                throw $exception;
+            }
+        }
+    }
 }

app/Actions/V5/Server/PushHostAgentToken.php

@@ -0,0 +1,101 @@
+<?php
+
+namespace App\Actions\V5\Server;
+
+use App\Models\PrivateKey;
+use App\Models\V5\Server;
+use Illuminate\Support\Facades\Process;
+use Lorisleiva\Actions\Concerns\AsAction;
+
+class PushHostAgentToken
+{
+    use AsAction;
+
+    /**
+     * Best-effort SSH push of a freshly minted host JWT to the on-host jwt path.
+     *
+     * coold re-reads the JWT file on every reconnect and flux drops the stream
+     * at the token's exp, so overwriting the file in place is enough for the
+     * next reconnect to pick up the new token — coold is intentionally NOT
+     * restarted here (a restart would force an unnecessary disconnect of a
+     * stream that is still valid on the current token).
+     *
+     * Mirrors V5BootstrapServerJob::enrollCooldIntoFlux for the write mechanics
+     * (printf %s <token> | sudo tee <path>; chmod 600) and RemoveBootstrapMarker
+     * for the SSH/temp-key mechanics. Returns whether the write succeeded;
+     * every failure path (missing key, SSH error, exception) resolves to false
+     * and always cleans up the temporary key file.
+     */
+    public function handle(Server $server, string $token): bool
+    {
+        $server->loadMissing('privateKey');
+
+        if (! $server->privateKey instanceof PrivateKey) {
+            return false;
+        }
+
+        $jwtPath = trim((string) config('coold.flux_host_jwt_path', '/etc/coolify/host-jwt'));
+
+        if ($jwtPath === '') {
+            $jwtPath = '/etc/coolify/host-jwt';
+        }
+
+        $jwtPath = str_replace(["\r", "\n"], '', $jwtPath);
+        $token = str_replace(["\r", "\n"], '', $token);
+
+        $keyDirectory = storage_path('app/ssh/keys');
+        if (! is_dir($keyDirectory)) {
+            mkdir($keyDirectory, 0700, true);
+        }
+
+        $keyLocation = tempnam($keyDirectory, 'v5_ssh_key_');
+        if ($keyLocation === false) {
+            return false;
+        }
+
+        file_put_contents($keyLocation, $server->privateKey->private_key);
+        chmod($keyLocation, 0600);
+
+        $tokenArgument = escapeshellarg($token);
+        $jwtPathArgument = $this->shellPathArg($jwtPath);
+        $script = <<<SH
+set -e
+SUDO=''
+if [ "\$(id -u)" != "0" ]; then SUDO='sudo'; fi
+\$SUDO mkdir -p /etc/coolify
+printf %s {$tokenArgument} | \$SUDO tee {$jwtPathArgument} >/dev/null
+\$SUDO chmod 600 {$jwtPathArgument}
+SH;
+
+        try {
+            $result = Process::timeout(30)->run([
+                'ssh',
+                '-o', 'BatchMode=yes',
+                '-o', 'LogLevel=ERROR',
+                '-o', 'StrictHostKeyChecking=no',
+                '-o', 'UserKnownHostsFile=/dev/null',
+                '-o', 'ConnectTimeout=10',
+                '-o', 'IdentitiesOnly=yes',
+                '-i', $keyLocation,
+                '-p', (string) $server->ssh_port,
+                "{$server->ssh_user}@{$server->host}",
+                $script,
+            ]);
+
+            return $result->successful();
+        } catch (\Throwable) {
+            return false;
+        } finally {
+            @unlink($keyLocation);
+        }
+    }
+
+    private function shellPathArg(string $value): string
+    {
+        if (preg_match('/^[A-Za-z0-9_\/:.,@%+=-]+$/', $value) === 1) {
+            return $value;
+        }
+
+        return escapeshellarg($value);
+    }
+}

app/Actions/V5/Server/RemoveBootstrapMarker.php

@@ -0,0 +1,75 @@
+<?php
+
+namespace App\Actions\V5\Server;
+
+use App\Models\PrivateKey;
+use App\Models\V5\Server;
+use App\Services\Flux\AgentTokenIssuer;
+use Illuminate\Support\Facades\Process;
+use Lorisleiva\Actions\Concerns\AsAction;
+
+class RemoveBootstrapMarker
+{
+    use AsAction;
+
+    /**
+     * Best-effort removal of the on-host bootstrap identity (marker, host JWT and
+     * Flux drop-in) so a re-added server can never silently adopt stale state.
+     *
+     * The host token jti is revoked first (a local DB write plus a best-effort
+     * push to the flux revocation store) so a captured or pre-copied token is
+     * recorded revoked even when the host is unreachable — see
+     * AgentTokenIssuer::revoke.
+     */
+    public function handle(Server $server): bool
+    {
+        app(AgentTokenIssuer::class)->revoke($server);
+
+        $server->loadMissing('privateKey');
+
+        if (! $server->privateKey instanceof PrivateKey) {
+            return false;
+        }
+
+        $keyDirectory = storage_path('app/ssh/keys');
+        if (! is_dir($keyDirectory)) {
+            mkdir($keyDirectory, 0700, true);
+        }
+
+        $keyLocation = tempnam($keyDirectory, 'v5_ssh_key_');
+        if ($keyLocation === false) {
+            return false;
+        }
+
+        file_put_contents($keyLocation, $server->privateKey->private_key);
+        chmod($keyLocation, 0600);
+
+        $script = implode("\n", [
+            "SUDO=''",
+            'if [ "$(id -u)" != "0" ]; then SUDO=\'sudo\'; fi',
+            '$SUDO rm -f /etc/coolify/v5-node.json /etc/coolify/host-jwt /etc/systemd/system/coold.service.d/10-flux.conf',
+        ]);
+
+        try {
+            $result = Process::timeout(15)->run([
+                'ssh',
+                '-o', 'BatchMode=yes',
+                '-o', 'LogLevel=ERROR',
+                '-o', 'StrictHostKeyChecking=no',
+                '-o', 'UserKnownHostsFile=/dev/null',
+                '-o', 'ConnectTimeout=10',
+                '-o', 'IdentitiesOnly=yes',
+                '-i', $keyLocation,
+                '-p', (string) $server->ssh_port,
+                "{$server->ssh_user}@{$server->host}",
+                $script,
+            ]);
+
+            return $result->successful();
+        } catch (\Throwable) {
+            return false;
+        } finally {
+            @unlink($keyLocation);
+        }
+    }
+}

app/Actions/V5/Server/SyncDevLimaServers.php

@@ -2,6 +2,7 @@
 
 namespace App\Actions\V5\Server;
 
+use App\Enums\V5\ServerStatus;
 use App\Models\PrivateKey;
 use App\Models\Team;
 use App\Models\User;
@@ -9,6 +10,14 @@ use App\Models\V5\Cluster;
 use App\Models\V5\Server;
 use Lorisleiva\Actions\Concerns\AsAction;
 
+/**
+ * Registers local Lima development VMs (provisioned by scripts/dev.sh) as
+ * cluster servers. They are intentionally seeded as Installed with
+ * last_bootstrapped_at already set but has_coold=false, so they skip the real
+ * bootstrap flow by design: V5BootstrapServerJob early-returns on a non-null
+ * last_bootstrapped_at, and V5ReconcileServersJob ignores them until
+ * something marks has_coold=true.
+ */
 class SyncDevLimaServers
 {
     use AsAction;
@@ -39,8 +48,6 @@ class SyncDevLimaServers
             'description' => 'Local Lima development cluster managed by scripts/dev.sh.',
         ]);
 
-        $capabilities = [];
-
         foreach ($servers as $server) {
             $wireguardManagementIp = $server['wireguard_management_ip'] ?? null;
             $values = [
@@ -49,8 +56,9 @@ class SyncDevLimaServers
                 'host' => $server['host'],
                 'ssh_user' => $server['ssh_user'],
                 'ssh_port' => $server['ssh_port'],
-                'status' => 'installed',
-                'capabilities' => $capabilities,
+                'status' => ServerStatus::Installed->value,
+                'has_coold' => false,
+                'is_ingress' => false,
                 'builder_enabled' => false,
                 'builder_capacity' => 0,
                 'node_address' => $wireguardManagementIp ?: $server['host'],