tiennm99/coolify
1
0 Fork 0
mirror of https://github.com/tiennm99/coolify.git synced 2026-10-13 08:15:24 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(servers): isolate cloud status checks from SSH checks

Browse Source 
Track provider state independently, skip SSH work for placeholder IPs, and clean up failed cloud server provisioning.
This commit is contained in:
Andras Bacsai committed 2026-07-11 22:52:10 +02:00
1 parent e01b8a057e
commit c8a332a3bc
22 files changed
+954 -180
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

tests/Feature/DigitalOceanApiTest.php
+104
View File
@@ -3,6 +3,7 @@
use App\Models\CloudProviderToken;
use App\Models\InstanceSettings;
use App\Models\PrivateKey;
use App\Models\Server;
use App\Models\Team;
use App\Models\User;
use Illuminate\Foundation\Testing\RefreshDatabase;
@@ -133,6 +134,109 @@ describe('POST /api/v1/servers/digitalocean', function () {
&& $request['user_data'] === '#cloud-config');
});
test('tracks the droplet with a placeholder IP when waiting for its IP fails', function () {
Http::fake([
'https://api.digitalocean.com/v2/account/keys' => Http::response([
'ssh_key' => ['id' => 123, 'fingerprint' => 'aa:bb:cc:dd'],
], 201),
'https://api.digitalocean.com/v2/account/keys*' => Http::response([
'ssh_keys' => [],
'links' => ['pages' => []],
], 200),
'https://api.digitalocean.com/v2/droplets' => Http::response([
'droplet' => [
'id' => 987,
'name' => 'waiting-for-ip',
'status' => 'new',
'networks' => ['v4' => [], 'v6' => []],
],
], 202),
'https://api.digitalocean.com/v2/droplets/987' => Http::response([
'message' => 'temporary provider failure',
], 500),
]);
$response = $this->withHeaders([
'Authorization' => 'Bearer '.$this->bearerToken,
'Content-Type' => 'application/json',
])->postJson('/api/v1/servers/digitalocean', [
'cloud_provider_token_id' => $this->digitalOceanToken->uuid,
'region' => 'nyc1',
'size' => 's-1vcpu-1gb',
'image' => 'ubuntu-24-04-x64',
'name' => 'waiting-for-ip',
'private_key_uuid' => $this->privateKey->uuid,
]);
$response->assertCreated();
$response->assertJsonFragment([
'digitalocean_droplet_id' => 987,
'ip' => Server::PLACEHOLDER_IP,
]);
$this->assertDatabaseHas('servers', [
'name' => 'waiting-for-ip',
'ip' => Server::PLACEHOLDER_IP,
'team_id' => $this->team->id,
'digitalocean_droplet_id' => 987,
'digitalocean_droplet_status' => 'new',
]);
});
test('deletes the droplet when local server persistence fails', function () {
Http::fake([
'https://api.digitalocean.com/v2/account/keys' => Http::response([
'ssh_key' => ['id' => 123, 'fingerprint' => 'aa:bb:cc:dd'],
], 201),
'https://api.digitalocean.com/v2/account/keys*' => Http::response([
'ssh_keys' => [],
'links' => ['pages' => []],
], 200),
'https://api.digitalocean.com/v2/droplets' => Http::response([
'droplet' => [
'id' => 987,
'name' => 'persistence-fails',
'status' => 'active',
'networks' => [
'v4' => [
['ip_address' => '203.0.113.10', 'type' => 'public'],
],
],
],
], 202),
'https://api.digitalocean.com/v2/droplets/987' => Http::response(null, 204),
]);
$eventDispatcher = Server::getEventDispatcher();
Server::setEventDispatcher(clone $eventDispatcher);
Server::created(function (): void {
throw new RuntimeException('local persistence failed');
});
try {
$response = $this->withHeaders([
'Authorization' => 'Bearer '.$this->bearerToken,
'Content-Type' => 'application/json',
])->postJson('/api/v1/servers/digitalocean', [
'cloud_provider_token_id' => $this->digitalOceanToken->uuid,
'region' => 'nyc1',
'size' => 's-1vcpu-1gb',
'image' => 'ubuntu-24-04-x64',
'name' => 'persistence-fails',
'private_key_uuid' => $this->privateKey->uuid,
]);
} finally {
Server::setEventDispatcher($eventDispatcher);
}
$response->assertServerError();
$this->assertDatabaseMissing('servers', [
'digitalocean_droplet_id' => 987,
]);
Http::assertSent(fn ($request) => $request->method() === 'DELETE'
&& $request->url() === 'https://api.digitalocean.com/v2/droplets/987');
});
test('validates required fields', function () {
$response = $this->withHeaders([
'Authorization' => 'Bearer '.$this->bearerToken,
tests/Feature/DigitalOceanServerCreationTest.php
+48
View File
@@ -113,6 +113,54 @@ it('updates the placeholder IP once the droplet reports one', function () {
]);
});
it('deletes the droplet when local server persistence fails', function () {
Http::fake([
'https://api.digitalocean.com/v2/account/keys' => Http::response([
'ssh_key' => ['id' => 123],
], 201),
'https://api.digitalocean.com/v2/account/keys*' => Http::response([
'ssh_keys' => [],
], 200),
'https://api.digitalocean.com/v2/droplets' => Http::response([
'droplet' => [
'id' => 555,
'status' => 'active',
'networks' => [
'v4' => [
['type' => 'public', 'ip_address' => '203.0.113.40'],
],
],
],
], 202),
'https://api.digitalocean.com/v2/droplets/555' => Http::response(null, 204),
]);
$eventDispatcher = Server::getEventDispatcher();
Server::setEventDispatcher(clone $eventDispatcher);
Server::created(function (): void {
throw new RuntimeException('local persistence failed');
});
try {
Livewire::test(ByDigitalOcean::class, ['selectedTokenUuid' => $this->digitalOceanToken->uuid])
->set('server_name', 'persistence-fails')
->set('selected_region', 'nyc1')
->set('selected_size', 's-1vcpu-1gb')
->set('selected_image', 'ubuntu-24-04-x64')
->set('private_key_id', $this->privateKey->id)
->call('submit')
->assertDispatched('error', 'local persistence failed');
} finally {
Server::setEventDispatcher($eventDispatcher);
}
$this->assertDatabaseMissing('servers', [
'digitalocean_droplet_id' => 555,
]);
Http::assertSent(fn ($request) => $request->method() === 'DELETE'
&& $request->url() === 'https://api.digitalocean.com/v2/droplets/555');
});
it('renders only the full width buy button at the bottom of the DigitalOcean form', function () {
Livewire::test(ByDigitalOcean::class)
->set('current_step', 2)
tests/Feature/ServerCloudProviderStatusCheckJobTest.php
+149
View File
@@ -0,0 +1,149 @@
<?php
use App\Jobs\ServerCloudProviderStatusCheckJob;
use App\Jobs\ServerConnectionCheckJob;
use App\Jobs\ServerManagerJob;
use App\Models\CloudProviderToken;
use App\Models\InstanceSettings;
use App\Models\PrivateKey;
use App\Models\Server;
use App\Models\Team;
use Illuminate\Foundation\Testing\RefreshDatabase;
use Illuminate\Support\Carbon;
use Illuminate\Support\Facades\Http;
use Illuminate\Support\Facades\Queue;
uses(RefreshDatabase::class);
function createCloudServerForStatusCheckJobTest(string $provider, array $attributes = []): Server
{
$team = Team::factory()->create();
$privateKey = PrivateKey::factory()->create(['team_id' => $team->id]);
$token = CloudProviderToken::create([
'team_id' => $team->id,
'provider' => $provider,
'token' => "test-{$provider}-token",
'name' => ucfirst($provider),
]);
return Server::factory()->create(array_merge([
'team_id' => $team->id,
'private_key_id' => $privateKey->id,
'cloud_provider_token_id' => $token->id,
'ip' => Server::PLACEHOLDER_IP,
], $attributes));
}
beforeEach(function () {
InstanceSettings::forceCreate([
'id' => 0,
'instance_timezone' => 'UTC',
]);
Carbon::setTestNow('2026-07-11 12:00:00');
});
afterEach(function () {
Carbon::setTestNow();
});
it('syncs provider state for a placeholder server without scheduling SSH', function () {
$server = createCloudServerForStatusCheckJobTest('digitalocean', [
'digitalocean_droplet_id' => 987,
'digitalocean_droplet_status' => 'new',
]);
Queue::fake();
(new ServerManagerJob)->handle();
Queue::assertPushed(ServerCloudProviderStatusCheckJob::class, fn (ServerCloudProviderStatusCheckJob $job) => $job->server->is($server));
Queue::assertNotPushed(ServerConnectionCheckJob::class);
});
it('backfills a DigitalOcean placeholder without changing reachability', function () {
$server = createCloudServerForStatusCheckJobTest('digitalocean', [
'digitalocean_droplet_id' => 987,
'digitalocean_droplet_status' => 'new',
]);
$server->settings->update([
'is_reachable' => true,
'is_usable' => true,
]);
Http::fake([
'https://api.digitalocean.com/v2/droplets/987' => Http::response([
'droplet' => [
'id' => 987,
'status' => 'active',
'networks' => [
'v4' => [
['type' => 'public', 'ip_address' => '203.0.113.10'],
],
],
],
]),
]);
(new ServerCloudProviderStatusCheckJob($server))->handle();
$server->refresh();
expect($server->ip)->toBe('203.0.113.10')
->and($server->digitalocean_droplet_status)->toBe('active')
->and((bool) $server->settings->fresh()->is_reachable)->toBeTrue()
->and((bool) $server->settings->fresh()->is_usable)->toBeTrue();
});
it('backfills a Hetzner placeholder without changing reachability', function () {
$server = createCloudServerForStatusCheckJobTest('hetzner', [
'hetzner_server_id' => 123,
'hetzner_server_status' => 'starting',
]);
$server->settings->update([
'is_reachable' => true,
'is_usable' => true,
]);
Http::fake([
'https://api.hetzner.cloud/v1/servers/123' => Http::response([
'server' => [
'id' => 123,
'status' => 'running',
'public_net' => [
'ipv4' => ['ip' => '198.51.100.20'],
],
],
]),
]);
(new ServerCloudProviderStatusCheckJob($server))->handle();
$server->refresh();
expect($server->ip)->toBe('198.51.100.20')
->and($server->hetzner_server_status)->toBe('running')
->and((bool) $server->settings->fresh()->is_reachable)->toBeTrue()
->and((bool) $server->settings->fresh()->is_usable)->toBeTrue();
});
it('only calls the API matching the cloud provider token', function () {
$server = createCloudServerForStatusCheckJobTest('digitalocean', [
'vultr_instance_id' => 'unrelated-vultr-instance',
'digitalocean_droplet_id' => 987,
'digitalocean_droplet_status' => 'new',
]);
Http::fake([
'https://api.digitalocean.com/v2/droplets/987' => Http::response([
'droplet' => [
'id' => 987,
'status' => 'active',
'networks' => ['v4' => []],
],
]),
'https://api.vultr.com/*' => Http::response(status: 500),
]);
(new ServerCloudProviderStatusCheckJob($server))->handle();
Http::assertSentCount(1);
Http::assertSent(fn ($request) => $request->url() === 'https://api.digitalocean.com/v2/droplets/987');
});
tests/Feature/ServerConnectionCheckIsolationTest.php
+80
View File
@@ -0,0 +1,80 @@
<?php
use App\Jobs\ServerConnectionCheckJob;
use App\Models\CloudProviderToken;
use App\Models\PrivateKey;
use App\Models\Server;
use App\Models\Team;
use Illuminate\Foundation\Testing\RefreshDatabase;
use Illuminate\Support\Facades\Http;
use Illuminate\Support\Facades\Process;
use Illuminate\Support\Facades\Storage;
uses(RefreshDatabase::class);
function createServerForConnectionIsolationTest(array $attributes = []): Server
{
$team = Team::factory()->create();
$privateKey = PrivateKey::factory()->create(['team_id' => $team->id]);
return Server::factory()->create(array_merge([
'team_id' => $team->id,
'private_key_id' => $privateKey->id,
], $attributes));
}
beforeEach(function () {
Storage::fake('ssh-keys');
});
it('never attempts SSH to a placeholder address', function (string $placeholderIp) {
$server = createServerForConnectionIsolationTest(['ip' => $placeholderIp]);
$server->settings->update([
'is_reachable' => true,
'is_usable' => true,
]);
Process::fake();
(new ServerConnectionCheckJob($server, disableMux: false))->handle();
Process::assertNothingRan();
expect($server->fresh()->unreachable_count)->toBe(0)
->and((bool) $server->settings->fresh()->is_reachable)->toBeTrue()
->and((bool) $server->settings->fresh()->is_usable)->toBeTrue();
})->with([
'reserved provisioning address' => Server::PLACEHOLDER_IP,
'unspecified IPv4 address' => '0.0.0.0',
'unspecified IPv6 address' => '::',
]);
it('does not call cloud provider APIs during an SSH connection check', function () {
$server = createServerForConnectionIsolationTest(['ip' => '203.0.113.10']);
$token = CloudProviderToken::create([
'team_id' => $server->team_id,
'provider' => 'vultr',
'token' => 'test-vultr-token',
'name' => 'Vultr',
]);
$server->update([
'cloud_provider_token_id' => $token->id,
'vultr_instance_id' => 'instance-1',
'vultr_instance_status' => 'active',
]);
Http::fake([
'https://api.vultr.com/*' => Http::response([
'instance' => ['id' => 'instance-1', 'status' => 'active'],
]),
]);
Process::fake([
'*' => Process::result(
output: '{"Server":{"Version":"27.0.0"}}',
exitCode: 0,
),
]);
(new ServerConnectionCheckJob($server->fresh(), disableMux: false))->handle();
Http::assertNothingSent();
});
tests/Feature/VultrApiTest.php
+60
View File
@@ -1,11 +1,15 @@
<?php
use App\Http\Controllers\Api\VultrController;
use App\Models\CloudProviderToken;
use App\Models\InstanceSettings;
use App\Models\PrivateKey;
use App\Models\Server;
use App\Models\Team;
use App\Models\User;
use Illuminate\Auth\Access\AuthorizationException;
use Illuminate\Foundation\Testing\RefreshDatabase;
use Illuminate\Http\Request;
use Illuminate\Support\Facades\Http;
uses(RefreshDatabase::class);
@@ -165,6 +169,27 @@ describe('GET /api/v1/vultr/ssh-keys', function () {
});
});
test('member read token cannot use a stored Vultr token for metadata endpoints', function (string $endpoint) {
$member = User::factory()->create();
$this->team->members()->attach($member->id, ['role' => 'member']);
$memberToken = $member->createToken('member-read', ['read'])->plainTextToken;
Http::fake();
$response = $this->withHeaders([
'Authorization' => 'Bearer '.$memberToken,
'Content-Type' => 'application/json',
])->getJson($endpoint.'?cloud_provider_token_uuid='.$this->vultrToken->uuid);
$response->assertForbidden();
Http::assertNothingSent();
})->with([
'/api/v1/vultr/regions',
'/api/v1/vultr/plans',
'/api/v1/vultr/os',
'/api/v1/vultr/ssh-keys',
]);
describe('POST /api/v1/servers/vultr', function () {
test('creates a Vultr server', function () {
Http::fake([
@@ -223,6 +248,11 @@ describe('POST /api/v1/servers/vultr', function () {
});
test('waits for Vultr to assign a real public IP when creation returns placeholder IP', function () {
$createdIp = null;
Server::created(function (Server $server) use (&$createdIp): void {
$createdIp = (string) $server->ip;
});
Http::fake([
'https://api.vultr.com/v2/ssh-keys' => Http::response([
'ssh_key' => ['id' => 'key-1', 'ssh_key' => testPublicKey()],
@@ -262,6 +292,8 @@ describe('POST /api/v1/servers/vultr', function () {
$response->assertStatus(201);
$response->assertJsonFragment(['ip' => '1.2.3.4']);
expect($createdIp)->toBe(Server::PLACEHOLDER_IP);
$this->assertDatabaseHas('servers', [
'name' => 'test-server',
'ip' => '1.2.3.4',
@@ -269,6 +301,34 @@ describe('POST /api/v1/servers/vultr', function () {
]);
});
test('server creation authorizes access to the stored Vultr token', function () {
$member = User::factory()->create();
$this->team->members()->attach($member->id, ['role' => 'member']);
$this->actingAs($member);
$member->withAccessToken($member->createToken('member-all', ['*'])->accessToken);
$request = Request::create(
uri: '/api/v1/servers/vultr',
method: 'POST',
server: ['CONTENT_TYPE' => 'application/json'],
content: json_encode([
'cloud_provider_token_uuid' => $this->vultrToken->uuid,
'region' => 'ewr',
'plan' => 'vc2-1c-1gb',
'os_id' => 2284,
'name' => 'test-server',
'private_key_uuid' => $this->privateKey->uuid,
], JSON_THROW_ON_ERROR),
);
$request->setUserResolver(fn () => $member);
Http::fake();
expect(fn () => app(VultrController::class)->createServer($request))
->toThrow(AuthorizationException::class);
Http::assertNothingSent();
});
test('reuses existing matching Vultr SSH key', function () {
Http::fake([
'https://api.vultr.com/v2/ssh-keys*' => Http::response([
tests/Feature/VultrServerLifecycleTest.php
+24 -2
View File
@@ -123,6 +123,8 @@ it('updates placeholder server IP when Vultr status refresh returns assigned pub
});
it('does not overwrite an established server IP during Vultr status refresh', function () {
$this->server->update(['ip' => '198.51.100.20']);
Http::fake([
'https://api.vultr.com/v2/instances/instance-1' => Http::response([
'instance' => [
@@ -135,12 +137,12 @@ it('does not overwrite an established server IP during Vultr status refresh', fu
Livewire::test(Show::class, ['server_uuid' => $this->server->uuid])
->call('checkVultrInstanceStatus', true)
->assertSet('ip', '1.2.3.4')
->assertSet('ip', '198.51.100.20')
->assertSet('vultrInstanceStatus', 'active');
$this->assertDatabaseHas('servers', [
'id' => $this->server->id,
'ip' => '1.2.3.4',
'ip' => '198.51.100.20',
'vultr_instance_status' => 'active',
]);
});
@@ -270,6 +272,26 @@ it('starts a Vultr instance', function () {
Http::assertSent(fn ($request) => $request->url() === 'https://api.vultr.com/v2/instances/instance-1/start');
});
it('prevents members from starting a Vultr instance', function () {
$this->server->update(['vultr_instance_status' => 'stopped']);
$member = User::factory()->create();
$this->team->members()->attach($member->id, ['role' => 'member']);
$this->actingAs($member);
session(['currentTeam' => $this->team]);
Http::fake([
'https://api.vultr.com/v2/instances/instance-1/start' => Http::response([], 204),
]);
Livewire::test(Show::class, ['server_uuid' => $this->server->uuid])
->call('startVultrInstance')
->assertDispatched('error');
expect($this->server->fresh()->vultr_instance_status)->toBe('stopped');
Http::assertNothingSent();
});
it('links a server to Vultr by matching IP', function () {
$unlinkedServer = Server::factory()->create([
'team_id' => $this->team->id,