fix(servers): retain cloud instances awaiting IPs

Persist DigitalOcean, Hetzner, and Vultr servers before public IP
assignment, then backfill placeholder addresses from provider state.

Treat partial Sentinel snapshots as non-authoritative and document the
destinations API with OpenAPI schemas.
This commit is contained in:
Andras Bacsai committed 2026-07-11 21:35:10 +02:00
1 parent d3fbb32c52
commit e01b8a057e
29 files changed
+1275 -59

No files matched your search

+20
View File
@@ -102,6 +102,26 @@ describe('GET /api/v1/servers/{server_uuid}/destinations', function () {
});
describe('POST /api/v1/servers/{server_uuid}/destinations', function () {
test('creates a standalone destination', function () {
$response = $this->withHeaders(destinationsApiHeaders($this->bearerToken))
->postJson("/api/v1/servers/{$this->server->uuid}/destinations", [
'name' => 'API Standalone',
'network' => 'api-standalone-network',
]);
$response->assertCreated()
->assertJson([
'name' => 'API Standalone',
'network' => 'api-standalone-network',
'type' => 'standalone',
'server_uuid' => $this->server->uuid,
]);
expect(StandaloneDocker::where('server_id', $this->server->id)
->where('network', 'api-standalone-network')
->exists())->toBeTrue();
});
test('requires a write token', function () {
$readOnlyToken = destinationsApiToken($this->user, $this->team, ['read']);
@@ -15,7 +15,10 @@ use Illuminate\Support\Str;
uses(RefreshDatabase::class);
beforeEach(function () {
InstanceSettings::updateOrCreate(['id' => 0], ['is_api_enabled' => true]);
InstanceSettings::unguarded(fn () => InstanceSettings::updateOrCreate(
['id' => 0],
['is_api_enabled' => true],
));
$this->team = Team::factory()->create();
@@ -1,10 +1,14 @@
<?php
use App\Livewire\Server\New\ByDigitalOcean;
use App\Models\CloudProviderToken;
use App\Models\InstanceSettings;
use App\Models\PrivateKey;
use App\Models\Server;
use App\Models\Team;
use App\Models\User;
use Illuminate\Foundation\Testing\RefreshDatabase;
use Illuminate\Support\Facades\Http;
use Livewire\Livewire;
uses(RefreshDatabase::class);
@@ -25,6 +29,88 @@ beforeEach(function () {
$this->actingAs($this->user);
session(['currentTeam' => $this->team]);
$this->digitalOceanToken = CloudProviderToken::create([
'team_id' => $this->team->id,
'provider' => 'digitalocean',
'token' => 'test-digitalocean-token',
'name' => 'Test DigitalOcean Token',
]);
$this->privateKey = PrivateKey::factory()->create(['team_id' => $this->team->id]);
});
function submitDigitalOceanServer(): void
{
Livewire::test(ByDigitalOcean::class, ['selectedTokenUuid' => test()->digitalOceanToken->uuid])
->assertSet('current_step', 2)
->set('server_name', 'test-do-server')
->set('selected_region', 'nyc1')
->set('selected_size', 's-1vcpu-1gb')
->set('selected_image', 'ubuntu-24-04-x64')
->set('private_key_id', test()->privateKey->id)
->call('submit')
->assertHasNoErrors();
}
it('persists the server with a placeholder IP when waiting for the droplet IP fails', function () {
Http::fake([
'https://api.digitalocean.com/v2/account/keys' => Http::response([
'ssh_key' => ['id' => 123],
], 201),
'https://api.digitalocean.com/v2/account/keys*' => Http::response([
'ssh_keys' => [],
], 200),
'https://api.digitalocean.com/v2/droplets' => Http::response([
'droplet' => ['id' => 555, 'status' => 'new'],
], 202),
'https://api.digitalocean.com/v2/droplets/555' => Http::response(['message' => 'server error'], 500),
]);
submitDigitalOceanServer();
$this->assertDatabaseHas('servers', [
'name' => 'test-do-server',
'ip' => Server::PLACEHOLDER_IP,
'team_id' => $this->team->id,
'cloud_provider_token_id' => $this->digitalOceanToken->id,
'digitalocean_droplet_id' => '555',
'digitalocean_droplet_status' => 'new',
]);
});
it('updates the placeholder IP once the droplet reports one', function () {
Http::fake([
'https://api.digitalocean.com/v2/account/keys' => Http::response([
'ssh_key' => ['id' => 123],
], 201),
'https://api.digitalocean.com/v2/account/keys*' => Http::response([
'ssh_keys' => [],
], 200),
'https://api.digitalocean.com/v2/droplets' => Http::response([
'droplet' => ['id' => 555, 'status' => 'new'],
], 202),
'https://api.digitalocean.com/v2/droplets/555' => Http::response([
'droplet' => [
'id' => 555,
'status' => 'active',
'networks' => [
'v4' => [
['type' => 'public', 'ip_address' => '203.0.113.40'],
],
],
],
], 200),
]);
submitDigitalOceanServer();
$this->assertDatabaseHas('servers', [
'name' => 'test-do-server',
'ip' => '203.0.113.40',
'digitalocean_droplet_id' => '555',
'digitalocean_droplet_status' => 'active',
]);
});
it('renders only the full width buy button at the bottom of the DigitalOcean form', function () {
+11
View File
@@ -100,6 +100,17 @@ test('MCP endpoint returns 403 when the token team has MCP disabled', function (
$response->assertJson(['message' => 'MCP server is disabled for this team.']);
});
test('MCP endpoint is enabled for teams by default', function () {
$defaultTeam = Team::factory()->create();
$this->user->teams()->attach($defaultTeam->id, ['role' => 'owner']);
session(['currentTeam' => $defaultTeam]);
$token = $this->user->createToken('mcp-read', ['read'])->plainTextToken;
$response = mcpListTools($token);
$response->assertOk();
});
test('MCP endpoint rejects unauthenticated requests', function () {
$response = mcpPost(['jsonrpc' => '2.0', 'id' => 1, 'method' => 'tools/list']);
$response->assertStatus(401);
+4 -1
View File
@@ -20,7 +20,10 @@ use Illuminate\Support\Facades\Log;
uses(RefreshDatabase::class);
beforeEach(function () {
InstanceSettings::create(['id' => 0, 'is_api_enabled' => true]);
InstanceSettings::unguarded(fn () => InstanceSettings::updateOrCreate(
['id' => 0],
['is_api_enabled' => true],
));
$this->team = Team::factory()->create();
$this->user = User::factory()->create();
@@ -126,3 +126,112 @@ function createPushUpdatePostgresql(Team $team, array $attributes = []): Standal
return $database;
}
test('partial sentinel snapshots do not mark missing databases as exited', function () {
$team = Team::factory()->create();
$database = createPushUpdatePostgresql($team, [
'status' => 'running:healthy',
'is_public' => true,
]);
$server = $database->destination->server;
Queue::fake();
$data = [
'snapshot' => [
'version' => 1,
'complete' => false,
],
'containers' => [
[
'name' => 'unrelated-container',
'state' => 'running',
'health_status' => 'healthy',
'labels' => [
'coolify.managed' => 'true',
],
],
],
];
$job = new PushServerUpdateJob($server, $data);
$job->handle();
$database->refresh();
expect($database->status)->toBe('running:healthy');
});
test('partial sentinel snapshots apply directly observed database status updates', function () {
$team = Team::factory()->create();
$database = createPushUpdatePostgresql($team, [
'status' => 'exited',
]);
$server = $database->destination->server;
Queue::fake();
$data = [
'snapshot' => [
'version' => 1,
'complete' => false,
],
'containers' => [
[
'name' => $database->uuid,
'state' => 'running',
'health_status' => 'healthy',
'labels' => [
'coolify.managed' => 'true',
'coolify.type' => 'database',
'com.docker.compose.service' => $database->uuid,
],
],
],
];
$job = new PushServerUpdateJob($server, $data);
$job->handle();
$database->refresh();
expect($database->status)->toBe('running:healthy');
});
test('partial sentinel snapshots do not trigger missing tcp proxy recovery', function () {
$team = Team::factory()->create();
$database = createPushUpdatePostgresql($team, [
'status' => 'exited',
'is_public' => true,
]);
$server = $database->destination->server;
Queue::fake();
$data = [
'snapshot' => [
'version' => 1,
'complete' => false,
],
'containers' => [
[
'name' => $database->uuid,
'state' => 'running',
'health_status' => 'healthy',
'labels' => [
'coolify.managed' => 'true',
'coolify.type' => 'database',
'com.docker.compose.service' => $database->uuid,
],
],
],
];
$job = new PushServerUpdateJob($server, $data);
$job->handle();
$database->refresh();
expect($database->status)->toBe('running:healthy');
Queue::assertNothingPushed();
});
@@ -196,3 +196,24 @@ it('rejects an invalid token without dispatching', function () use ($running) {
Queue::assertNotPushed(PushServerUpdateJob::class);
});
it('dispatches a complete snapshot after an identical partial snapshot', function () use ($running) {
$partialPayload = sentinelPayload($running()) + [
'snapshot' => [
'version' => 1,
'complete' => false,
],
];
$completePayload = sentinelPayload($running()) + [
'snapshot' => [
'version' => 1,
'complete' => true,
],
];
pushSentinel($this->token, $partialPayload)->assertOk();
pushSentinel($this->token, $completePayload)->assertOk();
Queue::assertPushed(PushServerUpdateJob::class, 2);
});
@@ -0,0 +1,118 @@
<?php
use App\Livewire\Server\New\ByHetzner;
use App\Models\CloudProviderToken;
use App\Models\InstanceSettings;
use App\Models\PrivateKey;
use App\Models\Server;
use App\Models\Team;
use App\Models\User;
use Illuminate\Foundation\Testing\RefreshDatabase;
use Illuminate\Support\Facades\Http;
use Livewire\Livewire;
uses(RefreshDatabase::class);
beforeEach(function () {
config([
'cache.default' => 'array',
'session.driver' => 'array',
]);
InstanceSettings::unguarded(fn () => InstanceSettings::query()->create([
'id' => 0,
]));
$this->team = Team::factory()->create();
$this->user = User::factory()->create();
$this->team->members()->attach($this->user->id, ['role' => 'owner']);
$this->actingAs($this->user);
session(['currentTeam' => $this->team]);
$this->hetznerToken = CloudProviderToken::create([
'team_id' => $this->team->id,
'provider' => 'hetzner',
'token' => 'test-hetzner-token',
'name' => 'Test Hetzner Token',
]);
$this->privateKey = PrivateKey::factory()->create(['team_id' => $this->team->id]);
});
it('persists the server with a placeholder IP when Hetzner has not assigned one yet', function () {
Http::fake([
'https://api.hetzner.cloud/v1/ssh_keys' => Http::response([
'ssh_key' => ['id' => 42, 'fingerprint' => 'ff:ff'],
], 201),
'https://api.hetzner.cloud/v1/ssh_keys*' => Http::response([
'ssh_keys' => [],
], 200),
'https://api.hetzner.cloud/v1/servers' => Http::response([
'server' => [
'id' => 777,
'status' => 'initializing',
'public_net' => [
'ipv4' => null,
'ipv6' => null,
],
],
], 201),
]);
Livewire::test(ByHetzner::class, ['selectedTokenUuid' => $this->hetznerToken->uuid])
->assertSet('current_step', 2)
->set('server_name', 'test-hetzner-server')
->set('selected_location', 'fsn1')
->set('selected_server_type', 'cx22')
->set('selected_image', 114690387)
->set('private_key_id', $this->privateKey->id)
->call('submit')
->assertHasNoErrors();
$this->assertDatabaseHas('servers', [
'name' => 'test-hetzner-server',
'ip' => Server::PLACEHOLDER_IP,
'team_id' => $this->team->id,
'cloud_provider_token_id' => $this->hetznerToken->id,
'hetzner_server_id' => '777',
'hetzner_server_status' => 'initializing',
]);
});
it('creates the server with the real IP when Hetzner assigns one immediately', function () {
Http::fake([
'https://api.hetzner.cloud/v1/ssh_keys' => Http::response([
'ssh_key' => ['id' => 42, 'fingerprint' => 'ff:ff'],
], 201),
'https://api.hetzner.cloud/v1/ssh_keys*' => Http::response([
'ssh_keys' => [],
], 200),
'https://api.hetzner.cloud/v1/servers' => Http::response([
'server' => [
'id' => 777,
'status' => 'running',
'public_net' => [
'ipv4' => ['ip' => '203.0.113.30'],
],
],
], 201),
]);
Livewire::test(ByHetzner::class, ['selectedTokenUuid' => $this->hetznerToken->uuid])
->assertSet('current_step', 2)
->set('server_name', 'test-hetzner-server')
->set('selected_location', 'fsn1')
->set('selected_server_type', 'cx22')
->set('selected_image', 114690387)
->set('private_key_id', $this->privateKey->id)
->call('submit')
->assertHasNoErrors();
$this->assertDatabaseHas('servers', [
'name' => 'test-hetzner-server',
'ip' => '203.0.113.30',
'hetzner_server_id' => '777',
'hetzner_server_status' => 'running',
]);
});
+2 -2
View File
@@ -15,8 +15,8 @@ function serviceExtraFieldsTestServiceWithApplicationImage(string $image): Servi
$team = Team::factory()->create();
$project = Project::factory()->create(['team_id' => $team->id]);
$environment = Environment::factory()->create(['project_id' => $project->id]);
$server = Server::factory()->create();
$destination = StandaloneDocker::factory()->create(['server_id' => $server->id]);
$server = Server::factory()->create(['team_id' => $team->id]);
$destination = StandaloneDocker::where('server_id', $server->id)->firstOrFail();
$service = Service::factory()->create([
'environment_id' => $environment->id,
+42 -2
View File
@@ -4,6 +4,7 @@ use App\Livewire\Server\New\ByVultr;
use App\Models\CloudProviderToken;
use App\Models\InstanceSettings;
use App\Models\PrivateKey;
use App\Models\Server;
use App\Models\Team;
use App\Models\User;
use Illuminate\Foundation\Testing\RefreshDatabase;
@@ -95,7 +96,7 @@ it('creates a Vultr server through the Livewire flow', function () {
'instance' => [
'id' => 'instance-1',
'label' => 'test-vultr-server',
'main_ip' => '1.2.3.4',
'main_ip' => '192.0.2.10',
'v6_main_ip' => '2001:db8::1',
'status' => 'pending',
],
@@ -115,7 +116,7 @@ it('creates a Vultr server through the Livewire flow', function () {
$this->assertDatabaseHas('servers', [
'name' => 'test-vultr-server',
'ip' => '1.2.3.4',
'ip' => '192.0.2.10',
'team_id' => $this->team->id,
'cloud_provider_token_id' => $this->vultrToken->id,
'vultr_instance_id' => 'instance-1',
@@ -132,6 +133,45 @@ it('creates a Vultr server through the Livewire flow', function () {
});
});
it('persists the server with a placeholder IP when Vultr has not assigned one yet', function () {
Http::fake([
'https://api.vultr.com/v2/ssh-keys' => Http::response([
'ssh_key' => ['id' => 'key-1', 'ssh_key' => vultrLivewireTestPublicKey()],
], 201),
'https://api.vultr.com/v2/ssh-keys*' => Http::response([
'ssh_keys' => [],
'meta' => ['links' => ['next' => null]],
], 200),
'https://api.vultr.com/v2/instances/instance-1' => Http::response(['error' => 'temporary error'], 500),
'https://api.vultr.com/v2/instances' => Http::response([
'instance' => [
'id' => 'instance-1',
'label' => 'test-vultr-server',
'main_ip' => '0.0.0.0',
'v6_main_ip' => '::',
'status' => 'pending',
],
], 202),
]);
Livewire::test(ByVultr::class, ['selectedTokenUuid' => $this->vultrToken->uuid])
->set('server_name', 'test-vultr-server')
->set('selected_region', 'ewr')
->set('selected_plan', 'vc2-1c-1gb')
->set('selected_os_id', 2284)
->set('private_key_id', $this->privateKey->id)
->call('submit')
->assertHasNoErrors();
$this->assertDatabaseHas('servers', [
'name' => 'test-vultr-server',
'ip' => Server::PLACEHOLDER_IP,
'team_id' => $this->team->id,
'vultr_instance_id' => 'instance-1',
'vultr_instance_status' => 'pending',
]);
});
it('requires IPv6 when public IPv4 is disabled', function () {
Livewire::test(ByVultr::class)
->set('selected_token_id', $this->vultrToken->id)
@@ -3,12 +3,14 @@
use App\Exceptions\DeploymentException;
use App\Jobs\ApplicationDeploymentJob;
use App\Models\Application;
use App\Models\ApplicationDeploymentQueue;
use App\Models\ApplicationSetting;
use App\Models\EnvironmentVariable;
use Illuminate\Foundation\Testing\RefreshDatabase;
use Illuminate\Support\Collection;
use Tests\TestCase;
uses(TestCase::class);
uses(TestCase::class, RefreshDatabase::class);
class TestableRailpackDeploymentJob extends ApplicationDeploymentJob
{
@@ -373,9 +375,9 @@ it('creates an empty build-time env file for railpack when there are no generate
]));
foreach ([
'application_deployment_queue' => new class
'application_deployment_queue' => new class extends ApplicationDeploymentQueue
{
public function addLogEntry(string $message, string $type = 'info', bool $hidden = false): void {}
public function addLogEntry(string $message, string $type = 'stdout', bool $hidden = false): void {}
},
'build_pack' => 'railpack',
'pull_request_id' => 0,
@@ -86,6 +86,7 @@ it('detects environment variable value changes without exposing secret values',
'is_buildtime' => false,
'is_runtime' => true,
'is_preview' => false,
'is_shown_once' => true,
'resourceable_type' => Application::class,
'resourceable_id' => $application->id,
]);
@@ -112,6 +113,7 @@ it('describes added environment variables as set without exposing secret values'
'is_buildtime' => false,
'is_runtime' => true,
'is_preview' => false,
'is_shown_once' => true,
'resourceable_type' => Application::class,
'resourceable_id' => $application->id,
]);
+13
View File
@@ -0,0 +1,13 @@
<?php
it('defines OpenAPI documentation for every destinations endpoint', function () {
$controller = file_get_contents(__DIR__.'/../../app/Http/Controllers/Api/DestinationsController.php');
$model = file_get_contents(__DIR__.'/../../app/Models/StandaloneDocker.php');
expect($model)->toContain("schema: 'Destination'")
->and($controller)->toContain("operationId: 'list-destinations'")
->and($controller)->toContain("operationId: 'get-destination-by-uuid'")
->and($controller)->toContain("operationId: 'delete-destination-by-uuid'")
->and($controller)->toContain("operationId: 'list-server-destinations'")
->and($controller)->toContain("operationId: 'create-server-destination'");
});
+45 -3
View File
@@ -10,7 +10,7 @@ use Tests\TestCase;
uses(TestCase::class, RefreshDatabase::class);
function createDigitalOceanServerForStateTest(string $status = 'active'): Server
function createDigitalOceanServerForStateTest(string $status = 'active', array $attributes = []): Server
{
$team = Team::create([
'name' => 'Test Team',
@@ -24,13 +24,13 @@ function createDigitalOceanServerForStateTest(string $status = 'active'): Server
]);
$privateKey = PrivateKey::factory()->create(['team_id' => $team->id]);
return Server::factory()->create([
return Server::factory()->create(array_merge([
'team_id' => $team->id,
'private_key_id' => $privateKey->id,
'cloud_provider_token_id' => $token->id,
'digitalocean_droplet_id' => 987,
'digitalocean_droplet_status' => $status,
]);
], $attributes));
}
it('marks a DigitalOcean droplet as deleted when the provider returns 404', function () {
@@ -45,6 +45,48 @@ it('marks a DigitalOcean droplet as deleted when the provider returns 404', func
expect($server->fresh()->digitalocean_droplet_status)->toBe('deleted');
});
it('backfills a placeholder IP from the DigitalOcean droplet state', function () {
$server = createDigitalOceanServerForStateTest('new', ['ip' => Server::PLACEHOLDER_IP]);
Http::fake([
'https://api.digitalocean.com/v2/droplets/987' => Http::response([
'droplet' => [
'id' => 987,
'status' => 'active',
'networks' => [
'v4' => [
['type' => 'public', 'ip_address' => '203.0.113.10'],
],
],
],
], 200),
]);
expect($server->refreshDigitalOceanState())->toBe('active');
expect($server->fresh()->ip)->toBe('203.0.113.10');
});
it('does not overwrite a real IP from the DigitalOcean droplet state', function () {
$server = createDigitalOceanServerForStateTest('active', ['ip' => '198.51.100.20']);
Http::fake([
'https://api.digitalocean.com/v2/droplets/987' => Http::response([
'droplet' => [
'id' => 987,
'status' => 'active',
'networks' => [
'v4' => [
['type' => 'public', 'ip_address' => '203.0.113.10'],
],
],
],
], 200),
]);
$server->refreshDigitalOceanState();
expect($server->fresh()->ip)->toBe('198.51.100.20');
});
it('does not mark a DigitalOcean droplet as deleted on transient provider errors', function () {
$server = createDigitalOceanServerForStateTest();
+3 -4
View File
@@ -1,13 +1,12 @@
<?php
test('expanded navbar theme item cycles themes without a right side selector', function () {
test('navbar does not render the removed theme switcher', function () {
$navbar = file_get_contents(__DIR__.'/../../resources/views/components/navbar.blade.php');
expect(substr_count($navbar, '@click.stop="cycleTheme()"'))->toBe(2)
expect($navbar)->not->toContain('cycleTheme()')
->and($navbar)->not->toContain('aria-label="Theme switcher"')
->and($navbar)->not->toContain('ml-auto flex items-center gap-0.5')
->and($navbar)->not->toContain('@click.stop="setTheme(\'light\')"')
->and($navbar)->not->toContain('@click.stop="setTheme(\'system\')"')
->and($navbar)->not->toContain('@click.stop="setTheme(\'dark\')"')
->and($navbar)->toContain('<span class="text-left menu-item-label">Theme</span>');
->and($navbar)->not->toContain('<span class="text-left menu-item-label">Theme</span>');
});
+67
View File
@@ -0,0 +1,67 @@
<?php
use App\Models\PrivateKey;
use App\Models\Server;
use App\Models\Team;
use Illuminate\Foundation\Testing\RefreshDatabase;
use Tests\TestCase;
uses(TestCase::class, RefreshDatabase::class);
function createServerForPlaceholderIpTest(string $ip): Server
{
$team = Team::create([
'name' => 'Test Team',
'personal_team' => false,
]);
$privateKey = PrivateKey::factory()->create(['team_id' => $team->id]);
return Server::factory()->create([
'team_id' => $team->id,
'private_key_id' => $privateKey->id,
'ip' => $ip,
]);
}
it('detects placeholder IPs', function () {
$server = createServerForPlaceholderIpTest(Server::PLACEHOLDER_IP);
foreach ([Server::PLACEHOLDER_IP, '0.0.0.0', '::'] as $ip) {
$server->update(['ip' => $ip]);
expect($server->fresh()->hasPlaceholderIp())->toBeTrue();
}
});
it('does not treat a real IP as a placeholder', function () {
$server = createServerForPlaceholderIpTest('203.0.113.5');
expect($server->hasPlaceholderIp())->toBeFalse();
});
it('backfills a placeholder IP with the real address', function () {
$server = createServerForPlaceholderIpTest(Server::PLACEHOLDER_IP);
expect($server->backfillPlaceholderIp('203.0.113.5'))->toBeTrue();
expect($server->fresh()->ip)->toBe('203.0.113.5');
});
it('does not overwrite a real IP when backfilling', function () {
$server = createServerForPlaceholderIpTest('203.0.113.5');
expect($server->backfillPlaceholderIp('198.51.100.9'))->toBeFalse();
expect($server->fresh()->ip)->toBe('203.0.113.5');
});
it('ignores a missing IP when backfilling', function () {
$server = createServerForPlaceholderIpTest(Server::PLACEHOLDER_IP);
expect($server->backfillPlaceholderIp(null))->toBeFalse();
expect($server->fresh()->ip)->toBe(Server::PLACEHOLDER_IP);
});
it('skips servers with a placeholder IP in scheduled jobs', function () {
$server = createServerForPlaceholderIpTest(Server::PLACEHOLDER_IP);
expect($server->skipServer())->toBeTrue();
});