Merge remote-tracking branch 'origin/next' into v5-parallel-inertia-react

tests/Unit/Api/LogEndpointHelpersTest.php

@@ -18,6 +18,20 @@ it('normalizes requested log line counts', function () {
         ->and(normalizeLogLines('50000'))->toBe(10000);
 });
 
+it('normalizes service resource log line counts before invoking Docker', function (string $controller, mixed $lines, int $expectedLines) {
+    $source = file_get_contents(__DIR__."/../../../app/Http/Controllers/Api/{$controller}.php");
+
+    expect($source)->toContain('$lines = normalizeLogLines($request->query(\'lines\'));')
+        ->and(normalizeLogLines($lines))->toBe($expectedLines);
+})->with([
+    'application invalid value' => ['ServiceApplicationsController', 'invalid', 100],
+    'application negative value' => ['ServiceApplicationsController', '-5', 100],
+    'application value above limit' => ['ServiceApplicationsController', '50000', 10000],
+    'database invalid value' => ['ServiceDatabasesController', 'invalid', 100],
+    'database negative value' => ['ServiceDatabasesController', '-5', 100],
+    'database value above limit' => ['ServiceDatabasesController', '50000', 10000],
+]);
+
 it('parses show_timestamps query values as booleans', function () {
     if (! function_exists('parseLogTimestampFlag')) {
         expect(function_exists('parseLogTimestampFlag'))->toBeTrue();

tests/Unit/ApplicationDeploymentRailpackConfigTest.php

@@ -3,12 +3,14 @@
 use App\Exceptions\DeploymentException;
 use App\Jobs\ApplicationDeploymentJob;
 use App\Models\Application;
+use App\Models\ApplicationDeploymentQueue;
 use App\Models\ApplicationSetting;
 use App\Models\EnvironmentVariable;
+use Illuminate\Foundation\Testing\RefreshDatabase;
 use Illuminate\Support\Collection;
 use Tests\TestCase;
 
-uses(TestCase::class);
+uses(TestCase::class, RefreshDatabase::class);
 
 class TestableRailpackDeploymentJob extends ApplicationDeploymentJob
 {
@@ -373,9 +375,9 @@ it('creates an empty build-time env file for railpack when there are no generate
     ]));
 
     foreach ([
-        'application_deployment_queue' => new class
+        'application_deployment_queue' => new class extends ApplicationDeploymentQueue
         {
-            public function addLogEntry(string $message, string $type = 'info', bool $hidden = false): void {}
+            public function addLogEntry(string $message, string $type = 'stdout', bool $hidden = false): void {}
         },
         'build_pack' => 'railpack',
         'pull_request_id' => 0,

tests/Unit/ApplicationOpenApiTest.php

@@ -0,0 +1,27 @@
+<?php
+
+use App\Models\Application;
+use OpenApi\Attributes\Property;
+use OpenApi\Attributes\Schema;
+
+it('documents application settings under the settings property', function () {
+    $schema = (new ReflectionClass(Application::class))
+        ->getAttributes(Schema::class)[0]
+        ->newInstance();
+
+    $openApi = json_decode(
+        file_get_contents(__DIR__.'/../../openapi.json'),
+        true,
+        flags: JSON_THROW_ON_ERROR,
+    );
+
+    $settingsProperty = collect($schema->properties)
+        ->first(fn (mixed $property): bool => $property instanceof Property
+            && $property->property === 'settings');
+
+    expect($settingsProperty)
+        ->not->toBeNull()
+        ->and($openApi['components']['schemas']['Application']['properties'])
+        ->toHaveKey('settings')
+        ->not->toHaveKey('');
+});

tests/Unit/ClickhouseBackupCommandTest.php

@@ -0,0 +1,36 @@
+<?php
+
+use App\Models\StandaloneClickhouse;
+use App\Support\ClickhouseBackupCommand;
+
+test('clickhouse exposes scheduled backups', function () {
+    expect((new StandaloneClickhouse)->isBackupSolutionAvailable())->toBeTrue();
+});
+
+test('builds a native clickhouse backup command using container credentials', function () {
+    $commands = ClickhouseBackupCommand::make(
+        containerName: 'clickhouse-uuid',
+        database: 'analytics',
+        archiveName: 'backup-uuid.zip',
+        backupDirectory: '/data/coolify/backups/clickhouse',
+    );
+
+    expect($commands)->toHaveCount(3)
+        ->and($commands[0])->toBe("mkdir -p '/data/coolify/backups/clickhouse'")
+        ->and($commands[1])->toContain("docker exec 'clickhouse-uuid' clickhouse-client")
+        ->and($commands[1])->not->toContain('--password')
+        ->and($commands[1])->toContain("BACKUP DATABASE `analytics` TO File('")
+        ->and($commands[1])->toContain('backup-uuid.zip')
+        ->and($commands[2])->toBe("docker cp 'clickhouse-uuid:/var/lib/clickhouse/backups/backup-uuid.zip' '/data/coolify/backups/clickhouse/backup-uuid.zip'")
+        ->and(ClickhouseBackupCommand::cleanup('clickhouse-uuid', 'backup-uuid.zip'))
+        ->toBe("docker exec 'clickhouse-uuid' rm -f '/var/lib/clickhouse/backups/backup-uuid.zip'");
+});
+
+test('rejects unsafe clickhouse database names', function () {
+    expect(fn () => ClickhouseBackupCommand::make(
+        containerName: 'clickhouse-uuid',
+        database: 'analytics`; DROP DATABASE default; --',
+        archiveName: 'backup-uuid.zip',
+        backupDirectory: '/data/coolify/backups/clickhouse',
+    ))->toThrow(Exception::class);
+});

tests/Unit/DeploymentConfiguration/ApplicationConfigurationSnapshotTest.php

@@ -6,6 +6,7 @@ use App\Models\Environment;
 use App\Models\EnvironmentVariable;
 use App\Models\Project;
 use App\Models\Team;
+use App\Services\DeploymentConfiguration\ConfigurationDiffer;
 use Illuminate\Foundation\Testing\RefreshDatabase;
 use Illuminate\Support\Str;
 use Tests\TestCase;
@@ -78,6 +79,192 @@ it('detects redeploy-only domain changes', function () {
         ->and($change['new_full_value'])->toBe($domains);
 });
 
+it('detects Docker image reference changes as redeploy-only changes', function (string $field, string $label, string $newValue) {
+    $application = snapshotTestApplication([
+        'build_pack' => 'dockerimage',
+        'docker_registry_image_name' => 'ghcr.io/coollabsio/shoutrrr',
+        'docker_registry_image_tag' => '1.3.0-rc.4',
+    ]);
+    markSnapshotTestApplicationDeployed($application);
+
+    $application->update([$field => $newValue]);
+    $diff = $application->refresh()->pendingDeploymentConfigurationDiff();
+    $change = collect($diff->changes())->firstWhere('label', $label);
+
+    expect($diff->isChanged())->toBeTrue()
+        ->and($diff->requiresBuild())->toBeFalse()
+        ->and($change)->not->toBeNull()
+        ->and($change['old_display_value'])->not->toBe($change['new_display_value'])
+        ->and($change['new_display_value'])->toBe($newValue);
+})->with([
+    'image name' => ['docker_registry_image_name', 'Docker image', 'ghcr.io/coollabsio/coolify'],
+    'image tag' => ['docker_registry_image_tag', 'Docker image tag or hash', '1.3.0-rc.5'],
+]);
+
+it('detects deployment hook changes as redeploy-only changes', function (string $field, string $label, string $newValue) {
+    $application = snapshotTestApplication();
+    markSnapshotTestApplicationDeployed($application);
+
+    $application->update([$field => $newValue]);
+    $diff = $application->refresh()->pendingDeploymentConfigurationDiff();
+
+    expect($diff->requiresBuild())->toBeFalse()
+        ->and(collect($diff->changes())->pluck('label'))->toContain($label);
+})->with([
+    'pre-deployment command' => ['pre_deployment_command', 'Pre-deployment command', 'php artisan migrate --force'],
+    'pre-deployment container' => ['pre_deployment_command_container', 'Pre-deployment command container', 'web'],
+    'post-deployment command' => ['post_deployment_command', 'Post-deployment command', 'php artisan cache:clear'],
+    'post-deployment container' => ['post_deployment_command_container', 'Post-deployment command container', 'worker'],
+]);
+
+it('detects source integration changes as build changes', function (string $field, string $label, mixed $newValue) {
+    $application = snapshotTestApplication([
+        'source_id' => 1,
+        'source_type' => 'App\\Models\\GithubApp',
+    ]);
+    markSnapshotTestApplicationDeployed($application);
+
+    $application->forceFill([$field => $newValue])->save();
+    $diff = $application->refresh()->pendingDeploymentConfigurationDiff();
+
+    expect($diff->requiresBuild())->toBeTrue()
+        ->and(collect($diff->changes())->pluck('label'))->toContain($label);
+})->with([
+    'source ID' => ['source_id', 'Source ID', 2],
+    'source type' => ['source_type', 'Source type', 'App\\Models\\GitlabApp'],
+]);
+
+it('detects build-affecting application settings', function (string $field, string $label) {
+    $application = snapshotTestApplication();
+    $application->settings->update([$field => false]);
+    markSnapshotTestApplicationDeployed($application);
+
+    $application->settings->update([$field => true]);
+    $diff = $application->refresh()->pendingDeploymentConfigurationDiff();
+
+    expect($diff->requiresBuild())->toBeTrue()
+        ->and(collect($diff->changes())->pluck('label'))->toContain($label);
+})->with([
+    'static site' => ['is_static', 'Static site'],
+    'single-page application' => ['is_spa', 'Single-page application'],
+    'Git submodules' => ['is_git_submodules_enabled', 'Git submodules'],
+    'Git LFS' => ['is_git_lfs_enabled', 'Git LFS'],
+    'shallow clone' => ['is_git_shallow_clone_enabled', 'Shallow clone'],
+    'environment variable sorting' => ['is_env_sorting_enabled', 'Sort environment variables'],
+]);
+
+it('detects runtime-affecting application settings as redeploy-only changes', function (string $field, string $label, mixed $newValue) {
+    $application = snapshotTestApplication();
+    $application->settings->update([$field => is_bool($newValue) ? ! $newValue : null]);
+    markSnapshotTestApplicationDeployed($application);
+
+    $application->settings->update([$field => $newValue]);
+    $diff = $application->refresh()->pendingDeploymentConfigurationDiff();
+
+    expect($diff->requiresBuild())->toBeFalse()
+        ->and(collect($diff->changes())->pluck('label'))->toContain($label);
+})->with([
+    'consistent container name' => ['is_consistent_container_name_enabled', 'Consistent container name', true],
+    'container label escaping' => ['is_container_label_escape_enabled', 'Escape container labels', false],
+    'container labels read-only' => ['is_container_label_readonly_enabled', 'Read-only container labels', false],
+    'log drain' => ['is_log_drain_enabled', 'Log drain', true],
+    'Swarm worker nodes' => ['is_swarm_only_worker_nodes', 'Swarm worker nodes only', true],
+    'stop grace period' => ['stop_grace_period', 'Stop grace period', 45],
+    'preserve repository' => ['is_preserve_repository_enabled', 'Preserve repository', true],
+]);
+
+it('classifies runtime options as redeploy-only and custom nginx configuration as build-affecting', function () {
+    $application = snapshotTestApplication();
+    markSnapshotTestApplicationDeployed($application);
+
+    $application->update(['custom_docker_run_options' => '--init']);
+
+    expect($application->refresh()->pendingDeploymentConfigurationDiff()->requiresBuild())->toBeFalse();
+
+    markSnapshotTestApplicationDeployed($application->refresh());
+    $application->update(['custom_nginx_configuration' => 'server { listen 80; }']);
+
+    expect($application->refresh()->pendingDeploymentConfigurationDiff()->requiresBuild())->toBeTrue();
+});
+
+it('keeps Docker run options compatible with older build-section snapshots', function () {
+    $application = snapshotTestApplication(['custom_docker_run_options' => '--init'])->refresh();
+    $previousSnapshot = $application->deploymentConfigurationSnapshot();
+    $buildItems = collect(data_get($previousSnapshot, 'sections.build.items'));
+
+    expect($buildItems->pluck('key'))->toContain('custom_docker_run_options');
+
+    data_set(
+        $previousSnapshot,
+        'sections.build.items',
+        $buildItems->map(function (array $item): array {
+            if ($item['key'] === 'custom_docker_run_options') {
+                $item['impact'] = 'build';
+            }
+
+            return $item;
+        })->all(),
+    );
+
+    expect(app(ConfigurationDiffer::class)->diff($previousSnapshot, $application->deploymentConfigurationSnapshot())->isChanged())->toBeFalse();
+
+    $application->update(['custom_docker_run_options' => '--rm']);
+
+    expect(app(ConfigurationDiffer::class)->diff($previousSnapshot, $application->refresh()->deploymentConfigurationSnapshot())->requiresBuild())->toBeFalse();
+});
+
+it('does not report newly tracked settings when an older snapshot omitted their default values', function () {
+    $application = snapshotTestApplication();
+    $application->settings->update(['stop_grace_period' => DEFAULT_STOP_GRACE_PERIOD_SECONDS]);
+    $currentSnapshot = $application->deploymentConfigurationSnapshot();
+    $introducedKeys = [
+        'is_static',
+        'is_spa',
+        'is_git_submodules_enabled',
+        'is_git_lfs_enabled',
+        'is_git_shallow_clone_enabled',
+        'is_env_sorting_enabled',
+        'is_consistent_container_name_enabled',
+        'is_container_label_escape_enabled',
+        'is_container_label_readonly_enabled',
+        'is_log_drain_enabled',
+        'is_swarm_only_worker_nodes',
+        'is_preserve_repository_enabled',
+        'stop_grace_period',
+    ];
+    $previousSnapshot = $currentSnapshot;
+
+    foreach (['build', 'runtime'] as $section) {
+        data_set(
+            $previousSnapshot,
+            "sections.{$section}.items",
+            collect(data_get($previousSnapshot, "sections.{$section}.items"))
+                ->reject(fn (array $item): bool => in_array($item['key'], $introducedKeys, true))
+                ->values()
+                ->all(),
+        );
+    }
+
+    expect(app(ConfigurationDiffer::class)->diff($previousSnapshot, $currentSnapshot)->isChanged())->toBeFalse();
+});
+
+it('accepts the historical environment sorting default in older snapshots', function () {
+    $application = snapshotTestApplication();
+    $application->settings->update(['is_env_sorting_enabled' => true]);
+    $currentSnapshot = $application->deploymentConfigurationSnapshot();
+    $previousSnapshot = $currentSnapshot;
+    data_set(
+        $previousSnapshot,
+        'sections.build.items',
+        collect(data_get($previousSnapshot, 'sections.build.items'))
+            ->reject(fn (array $item): bool => $item['key'] === 'is_env_sorting_enabled')
+            ->values()
+            ->all(),
+    );
+
+    expect(app(ConfigurationDiffer::class)->diff($previousSnapshot, $currentSnapshot)->isChanged())->toBeFalse();
+});
+
 it('detects environment variable value changes without exposing secret values', function () {
     $application = snapshotTestApplication();
     EnvironmentVariable::create([
@@ -86,6 +273,7 @@ it('detects environment variable value changes without exposing secret values',
         'is_buildtime' => false,
         'is_runtime' => true,
         'is_preview' => false,
+        'is_shown_once' => true,
         'resourceable_type' => Application::class,
         'resourceable_id' => $application->id,
     ]);
@@ -112,6 +300,7 @@ it('describes added environment variables as set without exposing secret values'
         'is_buildtime' => false,
         'is_runtime' => true,
         'is_preview' => false,
+        'is_shown_once' => true,
         'resourceable_type' => Application::class,
         'resourceable_id' => $application->id,
     ]);

tests/Unit/DestinationsOpenApiTest.php

@@ -0,0 +1,13 @@
+<?php
+
+it('defines OpenAPI documentation for every destinations endpoint', function () {
+    $controller = file_get_contents(__DIR__.'/../../app/Http/Controllers/Api/DestinationsController.php');
+    $model = file_get_contents(__DIR__.'/../../app/Models/StandaloneDocker.php');
+
+    expect($model)->toContain("schema: 'Destination'")
+        ->and($controller)->toContain("operationId: 'list-destinations'")
+        ->and($controller)->toContain("operationId: 'get-destination-by-uuid'")
+        ->and($controller)->toContain("operationId: 'delete-destination-by-uuid'")
+        ->and($controller)->toContain("operationId: 'list-server-destinations'")
+        ->and($controller)->toContain("operationId: 'create-server-destination'");
+});

tests/Unit/DigitalOceanDeleteServerTest.php

@@ -6,6 +6,7 @@ use App\Models\PrivateKey;
 use App\Models\Server;
 use App\Models\Team;
 use Illuminate\Foundation\Testing\RefreshDatabase;
+use Illuminate\Http\Client\RequestException;
 use Illuminate\Support\Facades\Http;
 use Tests\TestCase;
 
@@ -44,3 +45,59 @@ it('deletes the DigitalOcean droplet when requested', function () {
     Http::assertSent(fn ($request) => $request->method() === 'DELETE'
         && $request->url() === 'https://api.digitalocean.com/v2/droplets/987');
 });
+
+it('retains the server and surfaces a DigitalOcean deletion failure', function () {
+    Http::fake([
+        'https://api.digitalocean.com/v2/droplets/987' => Http::response([
+            'message' => 'deletion failed',
+        ], 500),
+    ]);
+
+    $team = Team::factory()->create();
+    $token = CloudProviderToken::factory()->create([
+        'team_id' => $team->id,
+        'provider' => 'digitalocean',
+        'token' => 'test-digitalocean-token',
+    ]);
+    $privateKey = PrivateKey::factory()->create(['team_id' => $team->id]);
+    $server = Server::factory()->create([
+        'team_id' => $team->id,
+        'private_key_id' => $privateKey->id,
+        'cloud_provider_token_id' => $token->id,
+        'digitalocean_droplet_id' => 987,
+    ]);
+    $server->delete();
+
+    expect(fn () => DeleteServer::run(
+        serverId: $server->id,
+        deleteFromDigitalOcean: true,
+        digitalOceanDropletId: 987,
+        cloudProviderTokenId: $token->id,
+        teamId: $team->id,
+    ))->toThrow(RequestException::class, 'status code 500');
+
+    expect(Server::withTrashed()->find($server->id))->not->toBeNull();
+});
+
+it('retains the server when no DigitalOcean token can delete the droplet', function () {
+    Http::preventStrayRequests();
+
+    $team = Team::factory()->create();
+    $privateKey = PrivateKey::factory()->create(['team_id' => $team->id]);
+    $server = Server::factory()->create([
+        'team_id' => $team->id,
+        'private_key_id' => $privateKey->id,
+        'cloud_provider_token_id' => null,
+        'digitalocean_droplet_id' => 987,
+    ]);
+    $server->delete();
+
+    expect(fn () => DeleteServer::run(
+        serverId: $server->id,
+        deleteFromDigitalOcean: true,
+        digitalOceanDropletId: 987,
+        teamId: $team->id,
+    ))->toThrow(RuntimeException::class, 'No DigitalOcean token found');
+
+    expect(Server::withTrashed()->find($server->id))->not->toBeNull();
+});

tests/Unit/DigitalOceanServerStateTest.php

@@ -10,7 +10,7 @@ use Tests\TestCase;
 
 uses(TestCase::class, RefreshDatabase::class);
 
-function createDigitalOceanServerForStateTest(string $status = 'active'): Server
+function createDigitalOceanServerForStateTest(string $status = 'active', array $attributes = []): Server
 {
     $team = Team::create([
         'name' => 'Test Team',
@@ -24,13 +24,13 @@ function createDigitalOceanServerForStateTest(string $status = 'active'): Server
     ]);
     $privateKey = PrivateKey::factory()->create(['team_id' => $team->id]);
 
-    return Server::factory()->create([
+    return Server::factory()->create(array_merge([
         'team_id' => $team->id,
         'private_key_id' => $privateKey->id,
         'cloud_provider_token_id' => $token->id,
         'digitalocean_droplet_id' => 987,
         'digitalocean_droplet_status' => $status,
-    ]);
+    ], $attributes));
 }
 
 it('marks a DigitalOcean droplet as deleted when the provider returns 404', function () {
@@ -45,6 +45,79 @@ it('marks a DigitalOcean droplet as deleted when the provider returns 404', func
     expect($server->fresh()->digitalocean_droplet_status)->toBe('deleted');
 });
 
+it('backfills a placeholder IP from the DigitalOcean droplet state', function () {
+    $server = createDigitalOceanServerForStateTest('new', ['ip' => Server::PLACEHOLDER_IP]);
+
+    Http::fake([
+        'https://api.digitalocean.com/v2/droplets/987' => Http::response([
+            'droplet' => [
+                'id' => 987,
+                'status' => 'active',
+                'networks' => [
+                    'v4' => [
+                        ['type' => 'public', 'ip_address' => '203.0.113.10'],
+                    ],
+                ],
+            ],
+        ], 200),
+    ]);
+
+    expect($server->refreshDigitalOceanState())->toBe('active');
+    expect($server->fresh()->ip)->toBe('203.0.113.10');
+});
+
+it('does not overwrite an administrator configured address from the DigitalOcean droplet state', function (string $configuredAddress) {
+    $server = createDigitalOceanServerForStateTest('active', ['ip' => $configuredAddress]);
+
+    Http::fake([
+        'https://api.digitalocean.com/v2/droplets/987' => Http::response([
+            'droplet' => [
+                'id' => 987,
+                'status' => 'active',
+                'networks' => [
+                    'v4' => [
+                        ['type' => 'public', 'ip_address' => '203.0.113.10'],
+                    ],
+                ],
+            ],
+        ], 200),
+    ]);
+
+    $server->refreshDigitalOceanState();
+    expect($server->fresh()->ip)->toBe($configuredAddress);
+})->with([
+    'public IPv4' => '198.51.100.20',
+    'private IPv4' => '10.10.0.12',
+    'IPv6' => '2001:db8::10',
+    'tunnel hostname' => 'server.internal.example.com',
+]);
+
+it('does not overwrite an address configured while DigitalOcean state is loading', function () {
+    $server = createDigitalOceanServerForStateTest('new', ['ip' => Server::PLACEHOLDER_IP]);
+
+    Http::fake(function () use ($server) {
+        Server::query()->findOrFail($server->id)->update([
+            'ip' => 'server.internal.example.com',
+        ]);
+
+        return Http::response([
+            'droplet' => [
+                'id' => 987,
+                'status' => 'active',
+                'networks' => [
+                    'v4' => [
+                        ['type' => 'public', 'ip_address' => '203.0.113.10'],
+                    ],
+                ],
+            ],
+        ], 200);
+    });
+
+    expect($server->refreshDigitalOceanState())->toBe('active')
+        ->and($server->fresh()->ip)->toBe('server.internal.example.com')
+        ->and($server->digitalocean_droplet_status)->toBe('active');
+});
+
 it('does not mark a DigitalOcean droplet as deleted on transient provider errors', function () {
     $server = createDigitalOceanServerForStateTest();
 

tests/Unit/DigitalOceanServiceTest.php

@@ -1,11 +1,59 @@
 <?php
 
+use App\Exceptions\RateLimitException;
 use App\Services\DigitalOceanService;
+use Illuminate\Http\Client\RequestException;
 use Illuminate\Support\Facades\Http;
 use Tests\TestCase;
 
 uses(TestCase::class);
 
+it('maps final DigitalOcean API error responses', function (int $status, string $message) {
+    Http::fake([
+        'https://api.digitalocean.com/v2/droplets' => Http::response(['message' => $message], $status),
+    ]);
+
+    $exception = null;
+
+    try {
+        (new DigitalOceanService('test-token'))->createDroplet([]);
+    } catch (Throwable $caught) {
+        $exception = $caught;
+    }
+
+    expect($exception)->toBeInstanceOf(Exception::class)
+        ->and($exception)->not->toBeInstanceOf(RequestException::class)
+        ->and($exception->getMessage())->toBe('DigitalOcean API error: '.$message)
+        ->and($exception->getCode())->toBe($status);
+
+    Http::assertSentCount(3);
+})->with([
+    'unauthorized' => [401, 'Unable to authenticate you'],
+    'unprocessable entity' => [422, 'Droplet name is invalid'],
+]);
+
+it('maps a final DigitalOcean rate-limit response with Retry-After', function () {
+    Http::fake([
+        'https://api.digitalocean.com/v2/droplets' => Http::sequence()
+            ->push(['message' => 'Temporary provider error'], 500)
+            ->push(['message' => 'Temporary provider error'], 500)
+            ->push(['message' => 'Too many requests'], 429, ['Retry-After' => '45']),
+    ]);
+
+    try {
+        (new DigitalOceanService('test-token'))->createDroplet([]);
+    } catch (RateLimitException $exception) {
+        expect($exception->getMessage())->toBe('Rate limit exceeded. Please try again later.')
+            ->and($exception->retryAfter)->toBe(45);
+
+        Http::assertSentCount(3);
+
+        return;
+    }
+
+    test()->fail('Expected a RateLimitException to be thrown.');
+});
+
 it('fetches paginated regions from DigitalOcean', function () {
     Http::fake([
         'https://api.digitalocean.com/v2/regions?page=1&per_page=50' => Http::response([

tests/Unit/EmailIdentityHelperTest.php

@@ -1,8 +1,12 @@
 <?php
 
 it('normalizes plus and dotted email identity variants', function () {
-    expect(normalize_email_identity('Ke.VinMcFadden+one@BTInternet.com'))->toBe('kevinmcfadden@btinternet.com');
-    expect(normalize_email_identity('k.e.v.i.n.m.c.f.a.d.d.e.n+two@btinternet.com'))->toBe('kevinmcfadden@btinternet.com');
+    expect(normalize_email_identity('Ke.VinMcFadden+one@Gmail.com'))->toBe('kevinmcfadden@gmail.com');
+    expect(normalize_email_identity('k.e.v.i.n.m.c.f.a.d.d.e.n+two@googlemail.com'))->toBe('kevinmcfadden@googlemail.com');
+});
+
+it('preserves dots and plus suffixes for ordinary email providers', function () {
+    expect(normalize_email_identity(' John.Smith+alerts@Example.com '))->toBe('john.smith+alerts@example.com');
 });
 
 it('returns null for blank or malformed email identities', function (?string $email) {

tests/Unit/NavbarThemeSwitcherTest.php

@@ -1,13 +1,12 @@
 <?php
 
-test('expanded navbar theme item cycles themes without a right side selector', function () {
+test('navbar does not render the removed theme switcher', function () {
     $navbar = file_get_contents(__DIR__.'/../../resources/views/components/navbar.blade.php');
 
-    expect(substr_count($navbar, '@click.stop="cycleTheme()"'))->toBe(2)
+    expect($navbar)->not->toContain('cycleTheme()')
         ->and($navbar)->not->toContain('aria-label="Theme switcher"')
-        ->and($navbar)->not->toContain('ml-auto flex items-center gap-0.5')
         ->and($navbar)->not->toContain('@click.stop="setTheme(\'light\')"')
         ->and($navbar)->not->toContain('@click.stop="setTheme(\'system\')"')
         ->and($navbar)->not->toContain('@click.stop="setTheme(\'dark\')"')
-        ->and($navbar)->toContain('<span class="text-left menu-item-label">Theme</span>');
+        ->and($navbar)->not->toContain('<span class="text-left menu-item-label">Theme</span>');
 });

tests/Unit/ServerBackoffTest.php

@@ -127,7 +127,7 @@ describe('shouldSkipDueToBackoff', function () {
 });
 
 describe('ServerConnectionCheckJob unreachable_count', function () {
-    it('marks Vultr servers unreachable when provider status is unavailable', function () {
+    it('marks servers unreachable when SSH is unavailable', function () {
         Event::fake([ServerReachabilityChanged::class]);
 
         $settings = Mockery::mock();
@@ -141,14 +141,12 @@ describe('ServerConnectionCheckJob unreachable_count', function () {
         $server->shouldReceive('getAttribute')->andReturnUsing(fn (string $key) => match ($key) {
             'settings' => $settings,
             'unreachable_notification_sent' => false,
-            'vultr_instance_id' => 'instance-1',
-            'cloudProviderToken' => (object) ['token' => 'test-token'],
+            'ip' => '203.0.113.10',
             'id' => 1,
             'name' => 'test-server',
             'unreachable_count' => 1,
             default => null,
         });
-        $server->shouldReceive('refreshVultrState')->once()->andReturn('stopped');
         $server->shouldReceive('increment')->with('unreachable_count')->once();
         $server->id = 1;
         $server->name = 'test-server';
@@ -158,22 +156,20 @@ describe('ServerConnectionCheckJob unreachable_count', function () {
         $job->handle();
     });
 
-    it('increments unreachable_count on timeout', function () {
+    it('does not change reachability on a job-level timeout', function () {
         Event::fake([ServerReachabilityChanged::class]);
 
         $settings = Mockery::mock();
         $settings->is_reachable = true;
-        $settings->shouldReceive('update')
-            ->with(['is_reachable' => false, 'is_usable' => false])
-            ->once();
+        $settings->shouldNotReceive('update');
 
         $server = Mockery::mock(Server::class)->makePartial()->shouldAllowMockingProtectedMethods();
         $server->shouldReceive('getAttribute')->with('settings')->andReturn($settings);
         $server->shouldReceive('getAttribute')->with('unreachable_notification_sent')->andReturn(false);
-        $server->shouldReceive('increment')->with('unreachable_count')->once();
+        $server->shouldNotReceive('increment');
         $server->id = 1;
         $server->name = 'test-server';
-        $server->unreachable_count = 1; // Will become 2 after increment in real code; mock keeps value as-is
+        $server->unreachable_count = 1;
 
         $job = new ServerConnectionCheckJob($server);
         $job->failed(new TimeoutExceededException);

tests/Unit/ServerPlaceholderIpTest.php

@@ -0,0 +1,125 @@
+<?php
+
+use App\Jobs\CleanupOrphanedPreviewContainersJob;
+use App\Jobs\ScheduledJobManager;
+use App\Models\PrivateKey;
+use App\Models\Server;
+use App\Models\Team;
+use Illuminate\Foundation\Testing\RefreshDatabase;
+use Tests\TestCase;
+
+uses(TestCase::class, RefreshDatabase::class);
+
+function createServerForPlaceholderIpTest(string $ip): Server
+{
+    $team = Team::create([
+        'name' => 'Test Team',
+        'personal_team' => false,
+    ]);
+    $privateKey = PrivateKey::factory()->create(['team_id' => $team->id]);
+
+    return Server::factory()->create([
+        'team_id' => $team->id,
+        'private_key_id' => $privateKey->id,
+        'ip' => $ip,
+    ]);
+}
+
+it('detects placeholder IPs', function () {
+    $server = createServerForPlaceholderIpTest(Server::PLACEHOLDER_IP);
+
+    foreach ([Server::PLACEHOLDER_IP, '0.0.0.0', '::'] as $ip) {
+        $server->update(['ip' => $ip]);
+
+        expect($server->fresh()->hasPlaceholderIp())->toBeTrue();
+    }
+});
+
+it('does not treat a real IP as a placeholder', function () {
+    $server = createServerForPlaceholderIpTest('203.0.113.5');
+
+    expect($server->hasPlaceholderIp())->toBeFalse();
+});
+
+it('backfills a placeholder IP with the real address', function () {
+    $server = createServerForPlaceholderIpTest(Server::PLACEHOLDER_IP);
+
+    expect($server->backfillPlaceholderIp('203.0.113.5'))->toBeTrue();
+    expect($server->fresh()->ip)->toBe('203.0.113.5');
+});
+
+it('does not overwrite a real IP when backfilling', function () {
+    $server = createServerForPlaceholderIpTest('203.0.113.5');
+
+    expect($server->backfillPlaceholderIp('198.51.100.9'))->toBeFalse();
+    expect($server->fresh()->ip)->toBe('203.0.113.5');
+});
+
+it('ignores a missing IP when backfilling', function () {
+    $server = createServerForPlaceholderIpTest(Server::PLACEHOLDER_IP);
+
+    expect($server->backfillPlaceholderIp(null))->toBeFalse();
+    expect($server->fresh()->ip)->toBe(Server::PLACEHOLDER_IP);
+});
+
+it('does not replace a placeholder with another placeholder', function (string $replacementIp) {
+    $server = createServerForPlaceholderIpTest(Server::PLACEHOLDER_IP);
+
+    expect($server->backfillPlaceholderIp($replacementIp))->toBeFalse();
+    expect($server->fresh()->ip)->toBe(Server::PLACEHOLDER_IP);
+})->with([
+    'unspecified IPv4 address' => '0.0.0.0',
+    'unspecified IPv6 address' => '::',
+]);
+
+it('does not overwrite an address configured after the placeholder model was loaded', function () {
+    $staleServer = createServerForPlaceholderIpTest(Server::PLACEHOLDER_IP);
+    $concurrentServer = Server::query()->findOrFail($staleServer->id);
+
+    $concurrentServer->update(['ip' => 'server.internal.example.com']);
+
+    expect($staleServer->backfillPlaceholderIp('203.0.113.10'))->toBeFalse()
+        ->and($staleServer->fresh()->ip)->toBe('server.internal.example.com');
+});
+
+it('skips servers with a placeholder IP in scheduled jobs', function () {
+    $server = createServerForPlaceholderIpTest(Server::PLACEHOLDER_IP);
+
+    expect($server->skipServer())->toBeTrue();
+});
+
+it('excludes every placeholder address from scheduled Docker cleanup', function () {
+    $realServer = createServerForPlaceholderIpTest('203.0.113.10');
+
+    foreach (Server::PLACEHOLDER_IPS as $placeholderIp) {
+        Server::factory()->create([
+            'team_id' => $realServer->team_id,
+            'private_key_id' => $realServer->private_key_id,
+            'ip' => $placeholderIp,
+        ]);
+    }
+
+    $method = new ReflectionMethod(ScheduledJobManager::class, 'getServersForCleanupQuery');
+    $servers = $method->invoke(new ScheduledJobManager)->get();
+
+    expect($servers->modelKeys())->toBe([$realServer->id]);
+});
+
+it('excludes every placeholder address from orphaned preview cleanup', function () {
+    $realServer = createServerForPlaceholderIpTest('203.0.113.10');
+    $realServer->settings->update(['is_reachable' => true, 'is_usable' => true]);
+
+    foreach (Server::PLACEHOLDER_IPS as $placeholderIp) {
+        $server = Server::factory()->create([
+            'team_id' => $realServer->team_id,
+            'private_key_id' => $realServer->private_key_id,
+            'ip' => $placeholderIp,
+        ]);
+        $server->settings->update(['is_reachable' => true, 'is_usable' => true]);
+    }
+
+    $method = new ReflectionMethod(CleanupOrphanedPreviewContainersJob::class, 'getServersToCheck');
+    $servers = $method->invoke(new CleanupOrphanedPreviewContainersJob);
+
+    expect($servers->modelKeys())->toBe([$realServer->id]);
+});

tests/Unit/ServiceApplicationsOpenApiTest.php

@@ -0,0 +1,32 @@
+<?php
+
+it('documents both supported methods for service application actions', function () {
+    $openApi = json_decode(
+        file_get_contents(__DIR__.'/../../openapi.json'),
+        true,
+        flags: JSON_THROW_ON_ERROR,
+    );
+
+    $actionPaths = [
+        '/services/{uuid}/applications/{app_uuid}/logs',
+        '/services/{uuid}/applications/{app_uuid}/start',
+        '/services/{uuid}/applications/{app_uuid}/restart',
+        '/services/{uuid}/applications/{app_uuid}/stop',
+    ];
+
+    foreach ($actionPaths as $path) {
+        expect($openApi['paths'][$path])
+            ->toHaveKeys(['get', 'post'])
+            ->and($openApi['paths'][$path]['post']['responses'])
+            ->toHaveKeys(['200', '400', '401', '404', '501']);
+    }
+
+    expect($openApi['paths'][$actionPaths[0]]['post']['responses']['200']['content']['application/json']['schema']['properties'])
+        ->toHaveKey('logs')
+        ->and($openApi['paths'][$actionPaths[1]]['post']['responses']['200']['content']['application/json']['schema']['properties'])
+        ->toHaveKey('message')
+        ->and($openApi['paths'][$actionPaths[2]]['post']['responses']['200']['content']['application/json']['schema']['properties'])
+        ->toHaveKey('message')
+        ->and($openApi['paths'][$actionPaths[3]]['post']['responses']['200']['content']['application/json']['schema']['properties'])
+        ->toHaveKey('message');
+});

tests/Unit/ServiceDatabasesOpenApiTest.php

@@ -0,0 +1,29 @@
+<?php
+
+it('documents service database management endpoints', function () {
+    $openApi = json_decode(
+        file_get_contents(__DIR__.'/../../openapi.json'),
+        true,
+        flags: JSON_THROW_ON_ERROR,
+    );
+
+    expect($openApi['paths'])
+        ->toHaveKey('/services/{uuid}/databases')
+        ->toHaveKey('/services/{uuid}/databases/{database_uuid}')
+        ->toHaveKey('/services/{uuid}/databases/{database_uuid}/logs')
+        ->toHaveKey('/services/{uuid}/databases/{database_uuid}/start')
+        ->toHaveKey('/services/{uuid}/databases/{database_uuid}/restart')
+        ->toHaveKey('/services/{uuid}/databases/{database_uuid}/stop')
+        ->and($openApi['paths']['/services/{uuid}/databases']['get'])
+        ->toBeArray()
+        ->and($openApi['paths']['/services/{uuid}/databases/{database_uuid}'])
+        ->toHaveKeys(['get', 'patch'])
+        ->and($openApi['paths']['/services/{uuid}/databases/{database_uuid}/logs']['get'])
+        ->toBeArray()
+        ->and($openApi['paths']['/services/{uuid}/databases/{database_uuid}/start']['post'])
+        ->toBeArray()
+        ->and($openApi['paths']['/services/{uuid}/databases/{database_uuid}/restart']['post'])
+        ->toBeArray()
+        ->and($openApi['paths']['/services/{uuid}/databases/{database_uuid}/stop']['post'])
+        ->toBeArray();
+});

tests/Unit/ServicesOpenApiTest.php

@@ -0,0 +1,28 @@
+<?php
+
+use App\Http\Controllers\Api\ServicesController;
+use OpenApi\Attributes\Patch;
+
+it('only documents fields supported by the update service endpoint', function () {
+    $method = new ReflectionMethod(ServicesController::class, 'update_by_uuid');
+    $patch = $method->getAttributes(Patch::class)[0]->newInstance();
+    $documentedProperties = $patch->requestBody->content[0]->schema->properties;
+
+    $openApi = json_decode(
+        file_get_contents(__DIR__.'/../../openapi.json'),
+        true,
+        flags: JSON_THROW_ON_ERROR,
+    );
+    $generatedProperties = $openApi['paths']['/services/{uuid}']['patch']['requestBody']['content']['application/json']['schema']['properties'];
+
+    $unsupportedFields = [
+        'project_uuid',
+        'environment_name',
+        'environment_uuid',
+        'server_uuid',
+        'destination_uuid',
+    ];
+
+    expect(array_keys($generatedProperties))->toBe(array_keys($documentedProperties))
+        ->and(array_intersect($unsupportedFields, array_keys($documentedProperties)))->toBeEmpty();
+});

tests/Unit/VultrDeleteServerTest.php

@@ -7,6 +7,7 @@ use App\Models\PrivateKey;
 use App\Models\Server;
 use App\Models\Team;
 use Illuminate\Foundation\Testing\RefreshDatabase;
+use Illuminate\Http\Client\RequestException;
 use Illuminate\Support\Facades\Http;
 use Tests\TestCase;
 
@@ -116,3 +117,52 @@ it('does not use another team Vultr token when deleting an instance', function (
 
     expect($request->header('Authorization'))->toBe(['Bearer test-vultr-token']);
 });
+
+it('retains the server and surfaces a Vultr deletion failure', function () {
+    Http::fake([
+        'https://api.vultr.com/v2/instances/instance-1' => Http::response([
+            'error' => 'deletion failed',
+        ], 500),
+    ]);
+
+    $server = Server::factory()->create([
+        'team_id' => $this->team->id,
+        'private_key_id' => $this->privateKey->id,
+        'cloud_provider_token_id' => $this->vultrToken->id,
+        'vultr_instance_id' => 'instance-1',
+    ]);
+    $server->delete();
+
+    expect(fn () => DeleteServer::run(
+        serverId: $server->id,
+        cloudProviderTokenId: $this->vultrToken->id,
+        teamId: $this->team->id,
+        deleteFromVultr: true,
+        vultrInstanceId: 'instance-1'
+    ))->toThrow(RequestException::class, 'status code 500');
+
+    expect(Server::withTrashed()->find($server->id))->not->toBeNull();
+});
+
+it('retains the server when no Vultr token can delete the instance', function () {
+    Http::preventStrayRequests();
+
+    $this->vultrToken->delete();
+
+    $server = Server::factory()->create([
+        'team_id' => $this->team->id,
+        'private_key_id' => $this->privateKey->id,
+        'cloud_provider_token_id' => null,
+        'vultr_instance_id' => 'instance-1',
+    ]);
+    $server->delete();
+
+    expect(fn () => DeleteServer::run(
+        serverId: $server->id,
+        teamId: $this->team->id,
+        deleteFromVultr: true,
+        vultrInstanceId: 'instance-1'
+    ))->toThrow(RuntimeException::class, 'No Vultr token found');
+
+    expect(Server::withTrashed()->find($server->id))->not->toBeNull();
+});

tests/Unit/VultrServiceTest.php

@@ -1,6 +1,8 @@
 <?php
 
+use App\Exceptions\RateLimitException;
 use App\Services\VultrService;
+use Illuminate\Http\Client\RequestException;
 use Illuminate\Support\Facades\Http;
 use Tests\TestCase;
 
@@ -10,6 +12,52 @@ beforeEach(function () {
     Http::preventStrayRequests();
 });
 
+it('maps final Vultr API error responses', function (int $status, string $message) {
+    Http::fake([
+        'https://api.vultr.com/v2/instances' => Http::response(['error' => $message], $status),
+    ]);
+
+    $exception = null;
+
+    try {
+        (new VultrService('fake-token'))->createInstance([]);
+    } catch (Throwable $caught) {
+        $exception = $caught;
+    }
+
+    expect($exception)->toBeInstanceOf(Exception::class)
+        ->and($exception)->not->toBeInstanceOf(RequestException::class)
+        ->and($exception->getMessage())->toBe('Vultr API error: '.$message)
+        ->and($exception->getCode())->toBe($status);
+
+    Http::assertSentCount(3);
+})->with([
+    'unauthorized' => [401, 'Unauthorized'],
+    'unprocessable entity' => [422, 'Invalid instance parameters'],
+]);
+
+it('maps a final Vultr rate-limit response with Retry-After', function () {
+    Http::fake([
+        'https://api.vultr.com/v2/instances' => Http::sequence()
+            ->push(['error' => 'Temporary provider error'], 500)
+            ->push(['error' => 'Temporary provider error'], 500)
+            ->push(['error' => 'Too many requests'], 429, ['Retry-After' => '30']),
+    ]);
+
+    try {
+        (new VultrService('fake-token'))->createInstance([]);
+    } catch (RateLimitException $exception) {
+        expect($exception->getMessage())->toBe('Rate limit exceeded. Please try again later.')
+            ->and($exception->retryAfter)->toBe(30);
+
+        Http::assertSentCount(3);
+
+        return;
+    }
+
+    test()->fail('Expected a RateLimitException to be thrown.');
+});
+
 it('gets instances from Vultr API', function () {
     Http::fake([
         'https://api.vultr.com/v2/instances*' => Http::response([