feat(servers): add cross-instance server migration (#11075)

2026-10-13 20:14:45 +00:00 · 2026-08-07 23:07:55 +02:00
1 parent c15e3b35fd
commit f9caa5250d
@@ -0,0 +1,181 @@
+<?php
+
+namespace App\Services\ServerTransfer;
+
+use Illuminate\Support\Facades\Crypt;
+use Illuminate\Validation\ValidationException;
+use RuntimeException;
+
+class ServerTransferBundle
+{
+    public const SCHEMA_VERSION = 1;
+
+    public const CLAIM_PATH = '/data/coolify/instance-claim.json';
+
+    public const MAILBOX_DIR = '/data/coolify/exports';
+
+    /**
+     * @param  array<string, mixed>  $payload
+     */
+    public static function wrap(array $payload): array
+    {
+        return array_merge([
+            'schema_version' => self::SCHEMA_VERSION,
+            'exported_at' => now()->toIso8601String(),
+            'export_id' => new_public_id(),
+        ], $payload);
+    }
+
+    /**
+     * @param  array<string, mixed>  $bundle
+     * @return array{valid: bool, errors: list<string>, warnings: list<string>}
+     */
+    public static function validate(array $bundle): array
+    {
+        $errors = [];
+        $warnings = [];
+
+        if (! isset($bundle['schema_version'])) {
+            $errors[] = 'Missing schema_version.';
+        } elseif ((int) $bundle['schema_version'] !== self::SCHEMA_VERSION) {
+            $errors[] = 'Unsupported schema_version '.$bundle['schema_version'].'. Expected '.self::SCHEMA_VERSION.'.';
+        }
+
+        if (! isset($bundle['export_id']) || ! is_string($bundle['export_id']) || $bundle['export_id'] === '') {
+            $errors[] = 'Missing export_id.';
+        }
+
+        if (! isset($bundle['server']) || ! is_array($bundle['server'])) {
+            $errors[] = 'Missing server payload.';
+        } else {
+            foreach (['uuid', 'name', 'ip', 'port', 'user'] as $field) {
+                if (! array_key_exists($field, $bundle['server'])) {
+                    $errors[] = "Missing server.{$field}.";
+                }
+            }
+        }
+
+        if (! isset($bundle['private_key']) || ! is_array($bundle['private_key'])) {
+            $errors[] = 'Missing private_key payload.';
+        } else {
+            if (blank(data_get($bundle, 'private_key.private_key'))) {
+                $errors[] = 'Missing private_key.private_key material.';
+            }
+        }
+
+        if (! isset($bundle['destinations']) || ! is_array($bundle['destinations'])) {
+            $errors[] = 'Missing destinations array.';
+        } elseif (count($bundle['destinations']) === 0) {
+            $warnings[] = 'Bundle has no destinations; a default destination will be used.';
+        }
+
+        if (! isset($bundle['projects']) || ! is_array($bundle['projects'])) {
+            $errors[] = 'Missing projects array.';
+        }
+
+        return [
+            'valid' => $errors === [],
+            'errors' => $errors,
+            'warnings' => $warnings,
+        ];
+    }
+
+    /**
+     * @param  array<string, mixed>  $bundle
+     */
+    public static function assertValid(array $bundle): void
+    {
+        $result = self::validate($bundle);
+        if (! $result['valid']) {
+            throw ValidationException::withMessages([
+                'bundle' => $result['errors'],
+            ]);
+        }
+    }
+
+    /**
+     * Encrypt an entire bundle for mailbox/API transport with a user passphrase.
+     *
+     * @param  array<string, mixed>  $bundle
+     * @return array{encrypted: true, schema_version: int, payload: string}
+     */
+    public static function encryptWithPassphrase(array $bundle, string $passphrase): array
+    {
+        if ($passphrase === '') {
+            throw new RuntimeException('Passphrase must not be empty.');
+        }
+
+        $json = json_encode($bundle, JSON_THROW_ON_ERROR);
+        $key = hash('sha256', $passphrase, true);
+        $iv = random_bytes(16);
+        $cipher = openssl_encrypt($json, 'AES-256-CBC', $key, OPENSSL_RAW_DATA, $iv);
+        if ($cipher === false) {
+            throw new RuntimeException('Failed to encrypt transfer bundle.');
+        }
+
+        $mac = hash_hmac('sha256', $iv.$cipher, $key);
+
+        return [
+            'encrypted' => true,
+            'schema_version' => self::SCHEMA_VERSION,
+            'payload' => base64_encode($iv.$mac.$cipher),
+        ];
+    }
+
+    /**
+     * @param  array<string, mixed>  $encrypted
+     * @return array<string, mixed>
+     */
+    public static function decryptWithPassphrase(array $encrypted, string $passphrase): array
+    {
+        if (! data_get($encrypted, 'encrypted')) {
+            throw new RuntimeException('Bundle is not encrypted.');
+        }
+
+        $raw = base64_decode((string) data_get($encrypted, 'payload'), true);
+        if ($raw === false || strlen($raw) < 48) {
+            throw new RuntimeException('Invalid encrypted payload.');
+        }
+
+        $key = hash('sha256', $passphrase, true);
+        $iv = substr($raw, 0, 16);
+        $mac = substr($raw, 16, 64);
+        $cipher = substr($raw, 80);
+        $expectedMac = hash_hmac('sha256', $iv.$cipher, $key);
+        if (! hash_equals($expectedMac, $mac)) {
+            throw new RuntimeException('Invalid passphrase or corrupted bundle.');
+        }
+
+        $json = openssl_decrypt($cipher, 'AES-256-CBC', $key, OPENSSL_RAW_DATA, $iv);
+        if ($json === false) {
+            throw new RuntimeException('Failed to decrypt transfer bundle.');
+        }
+
+        /** @var array<string, mixed> $bundle */
+        $bundle = json_decode($json, true, 512, JSON_THROW_ON_ERROR);
+
+        return $bundle;
+    }
+
+    /**
+     * Optionally re-seal secrets with Laravel's app key for intermediate storage.
+     * Prefer passphrase encryption for cross-instance transfers.
+     *
+     * @param  array<string, mixed>  $bundle
+     */
+    public static function sealWithAppKey(array $bundle): string
+    {
+        return Crypt::encryptString(json_encode($bundle, JSON_THROW_ON_ERROR));
+    }
+
+    /**
+     * @return array<string, mixed>
+     */
+    public static function unsealWithAppKey(string $sealed): array
+    {
+        /** @var array<string, mixed> $bundle */
+        $bundle = json_decode(Crypt::decryptString($sealed), true, 512, JSON_THROW_ON_ERROR);
+
+        return $bundle;
+    }
+}
@@ -0,0 +1,248 @@
+<?php
+
+namespace App\Services\ServerTransfer;
+
+use App\Models\Server;
+use Illuminate\Support\Facades\DB;
+use Illuminate\Support\Facades\Log;
+use RuntimeException;
+use Throwable;
+
+class ServerTransferClaimer
+{
+    /**
+     * Claim a managed host for this Coolify instance.
+     *
+     * Database ownership (metadata + Sentinel settings) is committed in one transaction so a
+     * mid-flight failure rolls back. Remote SSH claim-file writes happen after commit and are
+     * best-effort — they cannot participate in the DB transaction.
+     *
+     * @return array{
+     *     server_uuid: string,
+     *     claim: array<string, mixed>,
+     *     sentinel_rebound: bool,
+     *     claim_written: bool,
+     *     message: string
+     * }
+     */
+    public function claim(Server $server, bool $writeRemote = true, bool $rebindSentinel = true): array
+    {
+        if ($server->id === 0) {
+            throw new RuntimeException('Cannot claim the Coolify host itself.');
+        }
+
+        $instanceUrl = rtrim((string) (instanceSettings()->fqdn ?: config('app.url')), '/');
+        if ($instanceUrl === '') {
+            throw new RuntimeException('Instance URL (FQDN or APP_URL) must be set before claiming a server.');
+        }
+
+        $exportId = data_get($server->server_metadata, 'transfer.export_id');
+
+        $claim = [
+            'instance_url' => $instanceUrl,
+            'server_uuid' => $server->uuid,
+            'team_id' => $server->team_id,
+            'claimed_at' => now()->toIso8601String(),
+            'export_id' => $exportId,
+            'schema_version' => ServerTransferBundle::SCHEMA_VERSION,
+        ];
+
+        $result = DB::transaction(function () use ($server, $claim, $rebindSentinel, $instanceUrl) {
+            $server = Server::query()->with('settings')->lockForUpdate()->findOrFail($server->id);
+
+            $sentinelRebound = false;
+            if ($rebindSentinel && $server->settings) {
+                $server->settings->sentinel_custom_url = $instanceUrl;
+                $server->settings->ensureValidSentinelToken();
+                // Leave sentinel disabled until operator enables metrics; endpoint is ready.
+                $server->settings->save();
+                $sentinelRebound = true;
+            }
+
+            $metadata = $server->server_metadata ?? [];
+            $metadata['transfer'] = array_merge((array) data_get($metadata, 'transfer', []), [
+                'status' => 'claimed',
+                'claimed_at' => $claim['claimed_at'],
+                'claim' => $claim,
+                'claim_written' => false,
+                'sentinel_rebound' => $sentinelRebound,
+            ]);
+            $server->server_metadata = $metadata;
+            $server->save();
+
+            return [
+                'server_uuid' => $server->uuid,
+                'claim' => $claim,
+                'sentinel_rebound' => $sentinelRebound,
+                'claim_written' => false,
+            ];
+        });
+
+        // Remote host I/O is outside the transaction (cannot be rolled back with DB rows).
+        $claimWritten = false;
+        if ($writeRemote) {
+            $claimWritten = $this->writeClaimFile($server, $claim);
+            if ($claimWritten) {
+                $this->persistClaimWritten($server);
+            }
+        }
+
+        $result['claim_written'] = $claimWritten;
+        $result['message'] = $claimWritten
+            ? 'Server claimed. Ownership file written and Sentinel rebound to this instance.'
+            : 'Server claimed in Coolify. Remote ownership file was not written (SSH unavailable or skipped).';
+
+        return $result;
+    }
+
+    /**
+     * Mark a server as transferred away from this instance (source side).
+     * Disables automations to prevent dual management.
+     *
+     * All database writes run in one transaction so partial disable state cannot stick on failure.
+     * Local SSH key cache cleanup runs after commit (filesystem; not transactional).
+     *
+     * @return array{server_uuid: string, message: string}
+     */
+    public function markTransferred(Server $server, ?string $exportId = null, ?string $targetInstanceUrl = null): array
+    {
+        if ($server->id === 0) {
+            throw new RuntimeException('Cannot transfer the Coolify host itself.');
+        }
+
+        $result = DB::transaction(function () use ($server, $exportId, $targetInstanceUrl) {
+            $server = Server::query()->with('settings')->lockForUpdate()->findOrFail($server->id);
+
+            if ($server->settings) {
+                $server->settings->force_disabled = true;
+                $server->settings->is_sentinel_enabled = false;
+                $server->settings->save();
+            }
+
+            $metadata = $server->server_metadata ?? [];
+            $metadata['transfer'] = array_merge((array) data_get($metadata, 'transfer', []), [
+                'status' => 'transferred',
+                'export_id' => $exportId ?? data_get($metadata, 'transfer.export_id'),
+                'target_instance_url' => $targetInstanceUrl,
+                'transferred_at' => now()->toIso8601String(),
+            ]);
+            $server->server_metadata = $metadata;
+            $server->save();
+
+            return [
+                'server_uuid' => $server->uuid,
+                'message' => 'Server marked as transferred. Automations disabled on this instance.',
+            ];
+        });
+
+        $this->clearLocalSshArtifacts($server);
+
+        return $result;
+    }
+
+    private function persistClaimWritten(Server $server): void
+    {
+        DB::transaction(function () use ($server) {
+            $server = Server::query()->lockForUpdate()->find($server->id);
+            if (! $server) {
+                return;
+            }
+
+            $metadata = $server->server_metadata ?? [];
+            data_set($metadata, 'transfer.claim_written', true);
+            $server->server_metadata = $metadata;
+            $server->save();
+        });
+    }
+
+    /**
+     * Best-effort local SSH mux/key cleanup after transfer (filesystem; not part of DB rollback).
+     * forceDisableServer is idempotent for force_disabled and also clears cached SSH key material.
+     */
+    private function clearLocalSshArtifacts(Server $server): void
+    {
+        try {
+            $server->refresh();
+            $server->forceDisableServer();
+        } catch (Throwable $e) {
+            Log::warning('Failed to clear local SSH artifacts after transfer', [
+                'server_uuid' => $server->uuid,
+                'error' => $e->getMessage(),
+            ]);
+        }
+    }
+
+    /**
+     * Write export bundle to the managed host mailbox (air-gapped transfer).
+     *
+     * @param  array<string, mixed>  $bundle
+     * @return array{path: string, written: bool}
+     */
+    public function writeMailbox(Server $server, array $bundle, ?string $passphrase = null): array
+    {
+        ServerTransferBundle::assertValid($bundle);
+
+        $exportId = (string) data_get($bundle, 'export_id', new_public_id());
+        $filename = "server-transfer-{$exportId}.coolify.json";
+        $path = ServerTransferBundle::MAILBOX_DIR.'/'.$filename;
+
+        $payload = $passphrase
+            ? ServerTransferBundle::encryptWithPassphrase($bundle, $passphrase)
+            : $bundle;
+
+        $json = json_encode($payload, JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES);
+        $b64 = base64_encode($json);
+
+        $written = false;
+        try {
+            instant_remote_process([
+                'mkdir -p '.escapeshellarg(ServerTransferBundle::MAILBOX_DIR),
+                'echo '.escapeshellarg($b64).' | base64 -d > '.escapeshellarg($path),
+                'chmod 600 '.escapeshellarg($path),
+                'chown 9999:root '.escapeshellarg($path).' || true',
+            ], $server, true);
+            $written = true;
+        } catch (Throwable $e) {
+            Log::warning('Failed to write server transfer mailbox', [
+                'server_uuid' => $server->uuid,
+                'error' => $e->getMessage(),
+            ]);
+            if (! app()->runningUnitTests()) {
+                throw $e;
+            }
+        }
+
+        return [
+            'path' => $path,
+            'written' => $written,
+        ];
+    }
+
+    /**
+     * @param  array<string, mixed>  $claim
+     */
+    private function writeClaimFile(Server $server, array $claim): bool
+    {
+        $json = json_encode($claim, JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES);
+        $b64 = base64_encode($json);
+        $path = ServerTransferBundle::CLAIM_PATH;
+
+        try {
+            instant_remote_process([
+                'mkdir -p /data/coolify',
+                'echo '.escapeshellarg($b64).' | base64 -d > '.escapeshellarg($path),
+                'chmod 600 '.escapeshellarg($path),
+                'chown 9999:root '.escapeshellarg($path).' || true',
+            ], $server, true);
+
+            return true;
+        } catch (Throwable $e) {
+            Log::warning('Failed to write instance claim file', [
+                'server_uuid' => $server->uuid,
+                'error' => $e->getMessage(),
+            ]);
+
+            return false;
+        }
+    }
+}
@@ -0,0 +1,178 @@
+<?php
+
+namespace App\Services\ServerTransfer;
+
+use App\Models\Server;
+use Illuminate\Http\Client\ConnectionException;
+use Illuminate\Support\Facades\Http;
+use RuntimeException;
+use Throwable;
+
+class ServerTransferMigrator
+{
+    public function __construct(
+        private ServerTransferExporter $exporter,
+        private ServerTransferClaimer $claimer,
+    ) {}
+
+    /**
+     * One-shot migrate: export on this instance → import+claim on target → complete on this instance.
+     *
+     * @return array{
+     *     server_uuid: string,
+     *     export_id: string|null,
+     *     target_url: string,
+     *     import: array<string, mixed>,
+     *     complete: array<string, mixed>,
+     *     warnings: list<string>,
+     *     message: string
+     * }
+     */
+    public function migrate(
+        Server $server,
+        string $targetUrl,
+        string $targetToken,
+        bool $writeRemote = false,
+        bool $rebindSentinel = true,
+        bool $preserveUuids = true,
+        bool $adoptMode = true,
+    ): array {
+        if ($server->id === 0) {
+            throw new RuntimeException('The Coolify host (localhost) cannot be transferred between instances.');
+        }
+
+        $targetUrl = $this->normalizeTargetUrl($targetUrl);
+        $token = $this->normalizeToken($targetToken);
+
+        $bundle = $this->exporter->export($server, includeSensitive: true);
+        $exportId = data_get($bundle, 'export_id');
+        $warnings = array_values((array) data_get($bundle, 'warnings', []));
+
+        // Target import+claim runs in its own DB transaction on the remote instance.
+        // Source DB is unchanged until markTransferred below — so a failed remote import leaves source intact.
+        $importBody = $this->postImportToTarget(
+            targetUrl: $targetUrl,
+            token: $token,
+            bundle: $bundle,
+            writeRemote: $writeRemote,
+            rebindSentinel: $rebindSentinel,
+            preserveUuids: $preserveUuids,
+            adoptMode: $adoptMode,
+        );
+
+        if (is_array(data_get($importBody, 'warnings'))) {
+            $warnings = array_values(array_unique(array_merge($warnings, $importBody['warnings'])));
+        }
+
+        // Cross-instance 2PC is impossible: if complete fails after a successful import, the target
+        // already owns the server. Surface that clearly so the operator can retry complete only.
+        try {
+            $complete = $this->claimer->markTransferred(
+                $server,
+                exportId: is_string($exportId) ? $exportId : null,
+                targetInstanceUrl: $targetUrl,
+            );
+        } catch (Throwable $e) {
+            throw new RuntimeException(
+                "Server was imported on {$targetUrl}, but this instance could not mark it as transferred: {$e->getMessage()}. ".
+                'Retry complete (API: POST /api/v1/servers/{uuid}/complete) so automations stay disabled here. Do not re-import on the target.',
+                previous: $e
+            );
+        }
+
+        return [
+            'server_uuid' => $server->uuid,
+            'export_id' => is_string($exportId) ? $exportId : null,
+            'target_url' => $targetUrl,
+            'import' => $importBody,
+            'complete' => $complete,
+            'warnings' => $warnings,
+            'message' => 'Server migrated to '.$targetUrl.'. Automations disabled on this instance.',
+        ];
+    }
+
+    private function normalizeTargetUrl(string $targetUrl): string
+    {
+        $targetUrl = rtrim(trim($targetUrl), '/');
+        if ($targetUrl === '' || ! filter_var($targetUrl, FILTER_VALIDATE_URL)) {
+            throw new RuntimeException('A valid target instance URL is required (e.g. http://localhost:8001).');
+        }
+
+        // From inside Docker, localhost is this container — use the host gateway for peer instances.
+        if (file_exists('/.dockerenv') || is_file('/run/.containerenv')) {
+            $targetUrl = (string) preg_replace(
+                '#^(https?://)(localhost|127\.0\.0\.1)(?=[:/]|$)#i',
+                '$1host.docker.internal',
+                $targetUrl
+            );
+        }
+
+        return $targetUrl;
+    }
+
+    private function normalizeToken(string $targetToken): string
+    {
+        $token = trim($targetToken);
+        if (str_starts_with(strtolower($token), 'bearer ')) {
+            $token = trim(substr($token, 7));
+        }
+        if ($token === '') {
+            throw new RuntimeException('A target instance API token is required (root or write + create servers).');
+        }
+
+        return $token;
+    }
+
+    /**
+     * @param  array<string, mixed>  $bundle
+     * @return array<string, mixed>
+     */
+    private function postImportToTarget(
+        string $targetUrl,
+        string $token,
+        array $bundle,
+        bool $writeRemote,
+        bool $rebindSentinel,
+        bool $preserveUuids,
+        bool $adoptMode,
+    ): array {
+        $importUrl = $targetUrl.'/api/v1/servers/import';
+
+        try {
+            $response = Http::timeout(120)
+                ->acceptJson()
+                ->withToken($token)
+                ->asJson()
+                ->post($importUrl, [
+                    'bundle' => $bundle,
+                    'dry_run' => false,
+                    'preserve_uuids' => $preserveUuids,
+                    'adopt_mode' => $adoptMode,
+                    'claim' => true,
+                    'write_remote' => $writeRemote,
+                    'rebind_sentinel' => $rebindSentinel,
+                ]);
+        } catch (ConnectionException $e) {
+            throw new RuntimeException("Could not reach target instance at {$importUrl}: {$e->getMessage()}", previous: $e);
+        } catch (Throwable $e) {
+            throw new RuntimeException("Transfer to target failed: {$e->getMessage()}", previous: $e);
+        }
+
+        $importBody = $response->json();
+        if (! is_array($importBody)) {
+            $importBody = ['raw' => $response->body()];
+        }
+
+        if (! $response->successful()) {
+            $message = data_get($importBody, 'message')
+                ?? data_get($importBody, 'errors')
+                ?? $response->body();
+            if (is_array($message)) {
+                $message = json_encode($message);
+            }
+            throw new RuntimeException("Target import failed (HTTP {$response->status()}): {$message}");
+        }
+
+        return $importBody;
+    }
+}