fix(sentinel): reduce resource churn from health flaps

Ignore health status changes in Sentinel push deduplication when the container lifecycle state is unchanged.

Scope stale resource checks to Sentinel servers whose heartbeat is stale, and avoid refreshing resource last_online_at on unchanged statuses.

tests/Feature/PushServerUpdateJobLastOnlineTest.php

@@ -1,17 +1,19 @@
 <?php
 
 use App\Jobs\PushServerUpdateJob;
+use App\Models\Environment;
+use App\Models\Project;
 use App\Models\Server;
+use App\Models\StandaloneDocker;
 use App\Models\StandalonePostgresql;
 use App\Models\Team;
 use Illuminate\Foundation\Testing\RefreshDatabase;
 
 uses(RefreshDatabase::class);
 
-test('database last_online_at is updated when status unchanged', function () {
+test('database last_online_at is not updated when status is unchanged', function () {
     $team = Team::factory()->create();
-    $database = StandalonePostgresql::factory()->create([
-        'team_id' => $team->id,
+    $database = createPushUpdatePostgresql($team, [
         'status' => 'running:healthy',
         'last_online_at' => now()->subMinutes(5),
     ]);
@@ -40,15 +42,13 @@ test('database last_online_at is updated when status unchanged', function () {
 
     $database->refresh();
 
-    // last_online_at should be updated even though status didn't change
-    expect($database->last_online_at->greaterThan($oldLastOnline))->toBeTrue();
+    expect((string) $database->last_online_at)->toBe((string) $oldLastOnline);
     expect($database->status)->toBe('running:healthy');
 });
 
 test('database status is updated when container status changes', function () {
     $team = Team::factory()->create();
-    $database = StandalonePostgresql::factory()->create([
-        'team_id' => $team->id,
+    $database = createPushUpdatePostgresql($team, [
         'status' => 'exited',
     ]);
 
@@ -79,8 +79,7 @@ test('database status is updated when container status changes', function () {
 
 test('database is not marked exited when containers list is empty', function () {
     $team = Team::factory()->create();
-    $database = StandalonePostgresql::factory()->create([
-        'team_id' => $team->id,
+    $database = createPushUpdatePostgresql($team, [
         'status' => 'running:healthy',
     ]);
 
@@ -99,3 +98,31 @@ test('database is not marked exited when containers list is empty', function ()
     // Status should remain running, NOT be set to exited
     expect($database->status)->toBe('running:healthy');
 });
+
+function createPushUpdatePostgresql(Team $team, array $attributes = []): StandalonePostgresql
+{
+    $lastOnlineAt = $attributes['last_online_at'] ?? null;
+    unset($attributes['last_online_at']);
+
+    $server = Server::factory()->create(['team_id' => $team->id]);
+    $destination = StandaloneDocker::where('server_id', $server->id)->first()
+        ?? StandaloneDocker::factory()->create(['server_id' => $server->id]);
+    $project = Project::factory()->create(['team_id' => $team->id]);
+    $environment = Environment::factory()->create(['project_id' => $project->id]);
+
+    $database = StandalonePostgresql::create(array_merge([
+        'uuid' => (string) str()->uuid(),
+        'name' => 'postgres-'.str()->random(8),
+        'postgres_password' => 'secret',
+        'status' => 'exited',
+        'destination_id' => $destination->id,
+        'destination_type' => $destination->getMorphClass(),
+        'environment_id' => $environment->id,
+    ], $attributes));
+
+    if ($lastOnlineAt !== null) {
+        $database->forceFill(['last_online_at' => $lastOnlineAt])->saveQuietly();
+    }
+
+    return $database;
+}

tests/Feature/PushServerUpdateJobTest.php

@@ -4,17 +4,21 @@ use App\Jobs\PushServerUpdateJob;
 use App\Models\Server;
 use App\Models\Service;
 use App\Models\ServiceApplication;
+use App\Models\Team;
 use Illuminate\Foundation\Testing\RefreshDatabase;
 
 uses(RefreshDatabase::class);
 
 test('containers with empty service subId are skipped', function () {
-    $server = Server::factory()->create();
+    $team = Team::factory()->create();
+    $server = Server::factory()->create(['team_id' => $team->id]);
     $service = Service::factory()->create([
         'server_id' => $server->id,
     ]);
-    $serviceApp = ServiceApplication::factory()->create([
+    $serviceApp = ServiceApplication::create([
         'service_id' => $service->id,
+        'uuid' => (string) str()->uuid(),
+        'name' => 'app-'.str()->random(8),
     ]);
 
     $data = [
@@ -44,12 +48,15 @@ test('containers with empty service subId are skipped', function () {
 });
 
 test('containers with valid service subId are processed', function () {
-    $server = Server::factory()->create();
+    $team = Team::factory()->create();
+    $server = Server::factory()->create(['team_id' => $team->id]);
     $service = Service::factory()->create([
         'server_id' => $server->id,
     ]);
-    $serviceApp = ServiceApplication::factory()->create([
+    $serviceApp = ServiceApplication::create([
         'service_id' => $service->id,
+        'uuid' => (string) str()->uuid(),
+        'name' => 'app-'.str()->random(8),
     ]);
 
     $data = [

tests/Feature/ResourcesCheckTest.php

@@ -0,0 +1,78 @@
+<?php
+
+use App\Actions\Server\ResourcesCheck;
+use App\Models\Application;
+use App\Models\Environment;
+use App\Models\Project;
+use App\Models\Server;
+use App\Models\StandaloneDocker;
+use App\Models\Team;
+use Illuminate\Foundation\Testing\RefreshDatabase;
+use Illuminate\Support\Carbon;
+
+uses(RefreshDatabase::class);
+
+it('does not mark resources exited when sentinel is still reporting for the server', function () {
+    Carbon::setTestNow(Carbon::create(2026, 5, 27, 12, 0, 0, 'UTC'));
+    config(['constants.sentinel.resource_stale_seconds' => 300]);
+
+    $application = resourcesCheckApplication([
+        'last_online_at' => now()->subHour(),
+        'status' => 'running:healthy',
+    ], [
+        'sentinel_updated_at' => now()->subMinute(),
+    ]);
+
+    ResourcesCheck::run();
+
+    $application->refresh();
+
+    expect($application->status)->toBe('running:healthy');
+});
+
+it('marks resources exited when their sentinel server is stale', function () {
+    Carbon::setTestNow(Carbon::create(2026, 5, 27, 12, 0, 0, 'UTC'));
+    config(['constants.sentinel.resource_stale_seconds' => 300]);
+
+    $application = resourcesCheckApplication([
+        'last_online_at' => now()->subHour(),
+        'status' => 'running:healthy',
+    ], [
+        'sentinel_updated_at' => now()->subMinutes(10),
+    ]);
+
+    ResourcesCheck::run();
+
+    $application->refresh();
+
+    expect($application->status)->toBe('exited:unhealthy');
+});
+
+function resourcesCheckApplication(array $applicationAttributes = [], array $serverAttributes = []): Application
+{
+    $lastOnlineAt = $applicationAttributes['last_online_at'] ?? null;
+    unset($applicationAttributes['last_online_at']);
+
+    $team = Team::factory()->create();
+    $server = Server::factory()->create(array_merge([
+        'team_id' => $team->id,
+    ], $serverAttributes));
+    $server->settings()->update(['is_sentinel_enabled' => true]);
+
+    $destination = StandaloneDocker::where('server_id', $server->id)->first()
+        ?? StandaloneDocker::factory()->create(['server_id' => $server->id]);
+    $project = Project::factory()->create(['team_id' => $team->id]);
+    $environment = Environment::factory()->create(['project_id' => $project->id]);
+
+    $application = Application::factory()->create(array_merge([
+        'environment_id' => $environment->id,
+        'destination_id' => $destination->id,
+        'destination_type' => $destination->getMorphClass(),
+    ], $applicationAttributes));
+
+    if ($lastOnlineAt !== null) {
+        $application->forceFill(['last_online_at' => $lastOnlineAt])->saveQuietly();
+    }
+
+    return $application;
+}

tests/Feature/SentinelPushDeduplicationTest.php

@@ -138,6 +138,16 @@ it('dispatches the job when container state changes', function () use ($running)
     Queue::assertPushed(PushServerUpdateJob::class, 2);
 });
 
+it('ignores health status changes while container lifecycle state is unchanged', function () {
+    $healthy = [['name' => 'app-1', 'state' => 'running', 'health_status' => 'healthy']];
+    $unhealthy = [['name' => 'app-1', 'state' => 'running', 'health_status' => 'unhealthy']];
+
+    pushSentinel($this->token, sentinelPayload($healthy))->assertOk();
+    pushSentinel($this->token, sentinelPayload($unhealthy))->assertOk();
+
+    Queue::assertPushed(PushServerUpdateJob::class, 1);
+});
+
 it('ignores disk percentage changes (excluded from the hash)', function () use ($running) {
     pushSentinel($this->token, sentinelPayload($running(), diskPercentage: 42.0))->assertOk();
     pushSentinel($this->token, sentinelPayload($running(), diskPercentage: 88.0))->assertOk();