fix(sentinel): reduce resource churn from health flaps

Ignore health status changes in Sentinel push deduplication when the container lifecycle state is unchanged. Scope stale resource checks to Sentinel servers whose heartbeat is stale, and avoid refreshing resource last_online_at on unchanged statuses.
2026-10-18 12:14:07 +00:00 · 2026-05-27 16:48:38 +02:00
1 parent 1c5d5676ef
commit 626cfb4a22
@@ -3,9 +3,11 @@
 namespace App\Actions\Server;

 use App\Models\Application;
+use App\Models\Server;
 use App\Models\ServiceApplication;
 use App\Models\ServiceDatabase;
 use App\Models\StandaloneClickhouse;
+use App\Models\StandaloneDocker;
 use App\Models\StandaloneDragonfly;
 use App\Models\StandaloneKeydb;
 use App\Models\StandaloneMariadb;
@@ -13,6 +15,8 @@ use App\Models\StandaloneMongodb;
 use App\Models\StandaloneMysql;
 use App\Models\StandalonePostgresql;
 use App\Models\StandaloneRedis;
+use App\Models\SwarmDocker;
+use Illuminate\Support\Collection;
 use Lorisleiva\Actions\Concerns\AsAction;

 class ResourcesCheck
@@ -21,21 +25,73 @@ class ResourcesCheck

    public function handle()
    {
-        $seconds = 60;
+        $seconds = config('constants.sentinel.resource_stale_seconds', 300);
+        $staleServerIds = $this->staleSentinelServerIds($seconds);
+
+        if ($staleServerIds->isEmpty()) {
+            return;
+        }
+
+        [$standaloneDockerIds, $swarmDockerIds] = $this->destinationIdsForServers($staleServerIds);
+
        try {
-            Application::where('last_online_at', '<', now()->subSeconds($seconds))->update(['status' => 'exited']);
-            ServiceApplication::where('last_online_at', '<', now()->subSeconds($seconds))->update(['status' => 'exited']);
-            ServiceDatabase::where('last_online_at', '<', now()->subSeconds($seconds))->update(['status' => 'exited']);
-            StandalonePostgresql::where('last_online_at', '<', now()->subSeconds($seconds))->update(['status' => 'exited']);
-            StandaloneRedis::where('last_online_at', '<', now()->subSeconds($seconds))->update(['status' => 'exited']);
-            StandaloneMongodb::where('last_online_at', '<', now()->subSeconds($seconds))->update(['status' => 'exited']);
-            StandaloneMysql::where('last_online_at', '<', now()->subSeconds($seconds))->update(['status' => 'exited']);
-            StandaloneMariadb::where('last_online_at', '<', now()->subSeconds($seconds))->update(['status' => 'exited']);
-            StandaloneKeydb::where('last_online_at', '<', now()->subSeconds($seconds))->update(['status' => 'exited']);
-            StandaloneDragonfly::where('last_online_at', '<', now()->subSeconds($seconds))->update(['status' => 'exited']);
-            StandaloneClickhouse::where('last_online_at', '<', now()->subSeconds($seconds))->update(['status' => 'exited']);
+            Application::where(fn ($query) => $this->scopeDestination($query, $standaloneDockerIds, $swarmDockerIds))
+                ->where('status', 'not like', 'exited%')
+                ->update(['status' => 'exited']);
+
+            ServiceApplication::whereHas('service', fn ($query) => $query->whereIn('server_id', $staleServerIds))
+                ->where('status', 'not like', 'exited%')
+                ->update(['status' => 'exited']);
+
+            ServiceDatabase::whereHas('service', fn ($query) => $query->whereIn('server_id', $staleServerIds))
+                ->where('status', 'not like', 'exited%')
+                ->update(['status' => 'exited']);
+
+            collect([
+                StandalonePostgresql::class,
+                StandaloneRedis::class,
+                StandaloneMongodb::class,
+                StandaloneMysql::class,
+                StandaloneMariadb::class,
+                StandaloneKeydb::class,
+                StandaloneDragonfly::class,
+                StandaloneClickhouse::class,
+            ])->each(function (string $databaseClass) use ($standaloneDockerIds, $swarmDockerIds) {
+                $databaseClass::query()
+                    ->where(fn ($query) => $this->scopeDestination($query, $standaloneDockerIds, $swarmDockerIds))
+                    ->where('status', 'not like', 'exited%')
+                    ->update(['status' => 'exited']);
+            });
        } catch (\Throwable $e) {
            return handleError($e);
        }
    }
+
+    private function staleSentinelServerIds(int $seconds): Collection
+    {
+        return Server::query()
+            ->whereNotNull('sentinel_updated_at')
+            ->where('sentinel_updated_at', '<', now()->subSeconds($seconds))
+            ->whereHas('settings', fn ($query) => $query->where('is_sentinel_enabled', true))
+            ->pluck('id');
+    }
+
+    private function destinationIdsForServers(Collection $serverIds): array
+    {
+        return [
+            StandaloneDocker::whereIn('server_id', $serverIds)->pluck('id'),
+            SwarmDocker::whereIn('server_id', $serverIds)->pluck('id'),
+        ];
+    }
+
+    private function scopeDestination($query, Collection $standaloneDockerIds, Collection $swarmDockerIds): void
+    {
+        $query->where(function ($query) use ($standaloneDockerIds) {
+            $query->where('destination_type', StandaloneDocker::class)
+                ->whereIn('destination_id', $standaloneDockerIds);
+        })->orWhere(function ($query) use ($swarmDockerIds) {
+            $query->where('destination_type', SwarmDocker::class)
+                ->whereIn('destination_id', $swarmDockerIds);
+        });
+    }
 }
@@ -143,11 +143,13 @@ class SentinelController extends Controller
    /**
     * Build a stable hash of container state.
     *
-     * Covers [name, state, health_status] only — metrics and
-     * filesystem_usage_root are excluded on purpose (disk % churns constantly
-     * and would defeat the hash; the storage check is separately cache-gated
-     * inside PushServerUpdateJob). Sorted by name so container ordering from
-     * Sentinel does not affect the hash.
+     * Covers [name, state] only — metrics, filesystem_usage_root, and
+     * health_status are excluded on purpose. Disk % churns constantly, and
+     * health checks can flap between starting/healthy/unhealthy while the
+     * container lifecycle state remains unchanged. Both would otherwise defeat
+     * the hash and dispatch DB-heavy PushServerUpdateJob instances too often.
+     * The force window still refreshes full state periodically. Sorted by name
+     * so container ordering from Sentinel does not affect the hash.
     */
    private function containerStateHash(array $data): string
    {
@@ -155,7 +157,6 @@ class SentinelController extends Controller
            ->map(fn ($c) => [
                'name' => data_get($c, 'name'),
                'state' => data_get($c, 'state'),
-                'health_status' => data_get($c, 'health_status'),
            ])
            ->sortBy('name')
            ->values()
@@ -13,6 +13,16 @@ use App\Models\ApplicationPreview;
 use App\Models\Server;
 use App\Models\ServiceApplication;
 use App\Models\ServiceDatabase;
+use App\Models\StandaloneClickhouse;
+use App\Models\StandaloneDocker;
+use App\Models\StandaloneDragonfly;
+use App\Models\StandaloneKeydb;
+use App\Models\StandaloneMariadb;
+use App\Models\StandaloneMongodb;
+use App\Models\StandaloneMysql;
+use App\Models\StandalonePostgresql;
+use App\Models\StandaloneRedis;
+use App\Models\SwarmDocker;
 use App\Notifications\Container\ContainerRestarted;
 use App\Services\ContainerStatusAggregator;
 use App\Traits\CalculatesExcludedStatus;
@@ -25,6 +35,7 @@ use Illuminate\Queue\Middleware\WithoutOverlapping;
 use Illuminate\Queue\SerializesModels;
 use Illuminate\Support\Collection;
 use Illuminate\Support\Facades\Cache;
+use Illuminate\Support\Facades\DB;
 use Laravel\Horizon\Contracts\Silenced;

 class PushServerUpdateJob implements ShouldBeEncrypted, ShouldQueue, Silenced
@@ -46,6 +57,18 @@ class PushServerUpdateJob implements ShouldBeEncrypted, ShouldQueue, Silenced

    public Collection $services;

+    public Collection $applicationsById;
+
+    public Collection $previewsByKey;
+
+    public Collection $databasesByUuid;
+
+    public Collection $servicesById;
+
+    public Collection $serviceApplicationsById;
+
+    public Collection $serviceDatabasesById;
+
    public Collection $allApplicationIds;

    public Collection $allDatabaseUuids;
@@ -103,6 +126,12 @@ class PushServerUpdateJob implements ShouldBeEncrypted, ShouldQueue, Silenced
        $this->allTcpProxyUuids = collect();
        $this->allServiceApplicationIds = collect();
        $this->allServiceDatabaseIds = collect();
+        $this->applicationsById = collect();
+        $this->previewsByKey = collect();
+        $this->databasesByUuid = collect();
+        $this->servicesById = collect();
+        $this->serviceApplicationsById = collect();
+        $this->serviceDatabasesById = collect();
    }

    public function handle()
@@ -120,6 +149,12 @@ class PushServerUpdateJob implements ShouldBeEncrypted, ShouldQueue, Silenced
        $this->allTcpProxyUuids ??= collect();
        $this->allServiceApplicationIds ??= collect();
        $this->allServiceDatabaseIds ??= collect();
+        $this->applicationsById ??= collect();
+        $this->previewsByKey ??= collect();
+        $this->databasesByUuid ??= collect();
+        $this->servicesById ??= collect();
+        $this->serviceApplicationsById ??= collect();
+        $this->serviceDatabasesById ??= collect();

        // TODO: Swarm is not supported yet
        if (! $this->data) {
@@ -151,13 +186,16 @@ class PushServerUpdateJob implements ShouldBeEncrypted, ShouldQueue, Silenced
            return;
        }

-        $this->applications = $this->server->applications();
-        $this->databases = $this->server->databases();
-        $this->previews = $this->server->previews();
-        // Eager load service applications and databases to avoid N+1 queries
-        $this->services = $this->server->services()
-            ->with(['applications:id,service_id', 'databases:id,service_id'])
-            ->get();
+        $this->applications = $this->loadApplications();
+        $this->databases = $this->loadDatabases();
+        $this->previews = $this->loadPreviews();
+        $this->services = $this->loadServices();
+        $this->applicationsById = $this->applications->keyBy(fn ($application) => (string) $application->id);
+        $this->previewsByKey = $this->previews->keyBy(fn ($preview) => $preview->application_id.':'.$preview->pull_request_id);
+        $this->databasesByUuid = $this->databases->keyBy('uuid');
+        $this->servicesById = $this->services->keyBy(fn ($service) => (string) $service->id);
+        $this->serviceApplicationsById = $this->services->flatMap(fn ($service) => $service->applications)->keyBy(fn ($application) => (string) $application->id);
+        $this->serviceDatabasesById = $this->services->flatMap(fn ($service) => $service->databases)->keyBy(fn ($database) => (string) $database->id);

        $this->allApplicationIds = $this->applications->filter(function ($application) {
            return $application->additional_servers_count === 0;
@@ -170,9 +208,8 @@ class PushServerUpdateJob implements ShouldBeEncrypted, ShouldQueue, Silenced
        });
        $this->allDatabaseUuids = $this->databases->pluck('uuid');
        $this->allTcpProxyUuids = $this->databases->where('is_public', true)->pluck('uuid');
-        // Use eager-loaded relationships instead of querying in loop
-        $this->allServiceApplicationIds = $this->services->flatMap(fn ($service) => $service->applications->pluck('id'));
-        $this->allServiceDatabaseIds = $this->services->flatMap(fn ($service) => $service->databases->pluck('id'));
+        $this->allServiceApplicationIds = $this->serviceApplicationsById->keys();
+        $this->allServiceDatabaseIds = $this->serviceDatabasesById->keys();

        foreach ($this->containers as $container) {
            $containerStatus = data_get($container, 'state', 'exited');
@@ -286,6 +323,142 @@ class PushServerUpdateJob implements ShouldBeEncrypted, ShouldQueue, Silenced
        $this->checkLogDrainContainer();
    }

+    private function loadApplications(): Collection
+    {
+        [$standaloneDockerIds, $swarmDockerIds] = $this->serverDestinationIds();
+
+        $applications = Application::withoutGlobalScope('withRelations')
+            ->select([
+                'id',
+                'uuid',
+                'name',
+                'status',
+                'build_pack',
+                'docker_compose_raw',
+                'destination_id',
+                'destination_type',
+                'last_online_at',
+            ])
+            ->withCount('additional_servers')
+            ->where(fn ($query) => $this->scopeDestination($query, $standaloneDockerIds, $swarmDockerIds))
+            ->get();
+
+        $additionalApplicationIds = DB::table('additional_destinations')
+            ->where('server_id', $this->server->id)
+            ->pluck('application_id');
+
+        if ($additionalApplicationIds->isNotEmpty()) {
+            $applications = $applications->concat(
+                Application::withoutGlobalScope('withRelations')
+                    ->select([
+                        'id',
+                        'uuid',
+                        'name',
+                        'status',
+                        'build_pack',
+                        'docker_compose_raw',
+                        'destination_id',
+                        'destination_type',
+                        'last_online_at',
+                    ])
+                    ->withCount('additional_servers')
+                    ->whereIn('id', $additionalApplicationIds)
+                    ->get()
+            );
+        }
+
+        return $applications->unique('id')->values();
+    }
+
+    private function loadPreviews(): Collection
+    {
+        $applicationIds = $this->applications->pluck('id');
+
+        if ($applicationIds->isEmpty()) {
+            return collect();
+        }
+
+        return ApplicationPreview::query()
+            ->select([
+                'id',
+                'application_id',
+                'pull_request_id',
+                'status',
+                'last_online_at',
+            ])
+            ->whereIn('application_id', $applicationIds)
+            ->get();
+    }
+
+    private function loadServices(): Collection
+    {
+        return $this->server->services()
+            ->select([
+                'id',
+                'server_id',
+                'uuid',
+                'docker_compose_raw',
+            ])
+            ->with([
+                'applications:id,service_id,status,last_online_at',
+                'databases:id,service_id,status,last_online_at,is_public,name',
+            ])
+            ->get();
+    }
+
+    private function loadDatabases(): Collection
+    {
+        [$standaloneDockerIds, $swarmDockerIds] = $this->serverDestinationIds();
+        $databaseColumns = [
+            'id',
+            'uuid',
+            'name',
+            'status',
+            'is_public',
+            'destination_id',
+            'destination_type',
+            'last_online_at',
+            'restart_count',
+            'last_restart_at',
+            'last_restart_type',
+        ];
+
+        return collect([
+            StandalonePostgresql::class,
+            StandaloneRedis::class,
+            StandaloneMongodb::class,
+            StandaloneMysql::class,
+            StandaloneMariadb::class,
+            StandaloneKeydb::class,
+            StandaloneDragonfly::class,
+            StandaloneClickhouse::class,
+        ])->flatMap(function (string $databaseClass) use ($databaseColumns, $standaloneDockerIds, $swarmDockerIds) {
+            return $databaseClass::query()
+                ->select($databaseColumns)
+                ->where(fn ($query) => $this->scopeDestination($query, $standaloneDockerIds, $swarmDockerIds))
+                ->get();
+        })->filter(fn ($database) => data_get($database, 'name') !== 'coolify-db')->values();
+    }
+
+    private function serverDestinationIds(): array
+    {
+        return [
+            StandaloneDocker::where('server_id', $this->server->id)->pluck('id'),
+            SwarmDocker::where('server_id', $this->server->id)->pluck('id'),
+        ];
+    }
+
+    private function scopeDestination($query, Collection $standaloneDockerIds, Collection $swarmDockerIds): void
+    {
+        $query->where(function ($query) use ($standaloneDockerIds) {
+            $query->where('destination_type', StandaloneDocker::class)
+                ->whereIn('destination_id', $standaloneDockerIds);
+        })->orWhere(function ($query) use ($swarmDockerIds) {
+            $query->where('destination_type', SwarmDocker::class)
+                ->whereIn('destination_id', $swarmDockerIds);
+        });
+    }
+
    private function aggregateMultiContainerStatuses()
    {
        if ($this->applicationContainerStatuses->isEmpty()) {
@@ -293,7 +466,7 @@ class PushServerUpdateJob implements ShouldBeEncrypted, ShouldQueue, Silenced
        }

        foreach ($this->applicationContainerStatuses as $applicationId => $containerStatuses) {
-            $application = $this->applications->where('id', $applicationId)->first();
+            $application = $this->applicationsById->get((string) $applicationId);
            if (! $application) {
                continue;
            }
@@ -314,8 +487,6 @@ class PushServerUpdateJob implements ShouldBeEncrypted, ShouldQueue, Silenced
                if ($aggregatedStatus && $application->status !== $aggregatedStatus) {
                    $application->status = $aggregatedStatus;
                    $application->save();
-                } elseif ($aggregatedStatus) {
-                    $application->update(['last_online_at' => now()]);
                }

                continue;
@@ -330,8 +501,6 @@ class PushServerUpdateJob implements ShouldBeEncrypted, ShouldQueue, Silenced
            if ($aggregatedStatus && $application->status !== $aggregatedStatus) {
                $application->status = $aggregatedStatus;
                $application->save();
-            } elseif ($aggregatedStatus) {
-                $application->update(['last_online_at' => now()]);
            }
        }
    }
@@ -350,7 +519,7 @@ class PushServerUpdateJob implements ShouldBeEncrypted, ShouldQueue, Silenced
                continue;
            }

-            $service = $this->services->where('id', $serviceId)->first();
+            $service = $this->servicesById->get((string) $serviceId);
            if (! $service) {
                continue;
            }
@@ -358,9 +527,9 @@ class PushServerUpdateJob implements ShouldBeEncrypted, ShouldQueue, Silenced
            // Get the service sub-resource (ServiceApplication or ServiceDatabase)
            $subResource = null;
            if ($subType === 'application') {
-                $subResource = $service->applications->where('id', $subId)->first();
+                $subResource = $this->serviceApplicationsById->get((string) $subId);
            } elseif ($subType === 'database') {
-                $subResource = $service->databases->where('id', $subId)->first();
+                $subResource = $this->serviceDatabasesById->get((string) $subId);
            }

            if (! $subResource) {
@@ -382,8 +551,6 @@ class PushServerUpdateJob implements ShouldBeEncrypted, ShouldQueue, Silenced
                if ($aggregatedStatus && $subResource->status !== $aggregatedStatus) {
                    $subResource->status = $aggregatedStatus;
                    $subResource->save();
-                } elseif ($aggregatedStatus) {
-                    $subResource->update(['last_online_at' => now()]);
                }

                continue;
@@ -399,39 +566,31 @@ class PushServerUpdateJob implements ShouldBeEncrypted, ShouldQueue, Silenced
            if ($aggregatedStatus && $subResource->status !== $aggregatedStatus) {
                $subResource->status = $aggregatedStatus;
                $subResource->save();
-            } elseif ($aggregatedStatus) {
-                $subResource->update(['last_online_at' => now()]);
            }
        }
    }

    private function updateApplicationStatus(string $applicationId, string $containerStatus)
    {
-        $application = $this->applications->where('id', $applicationId)->first();
+        $application = $this->applicationsById->get((string) $applicationId);
        if (! $application) {
            return;
        }
        if ($application->status !== $containerStatus) {
            $application->status = $containerStatus;
            $application->save();
-        } else {
-            $application->update(['last_online_at' => now()]);
        }
    }

    private function updateApplicationPreviewStatus(string $applicationId, string $pullRequestId, string $containerStatus)
    {
-        $application = $this->previews->where('application_id', $applicationId)
-            ->where('pull_request_id', $pullRequestId)
-            ->first();
+        $application = $this->previewsByKey->get($applicationId.':'.$pullRequestId);
        if (! $application) {
            return;
        }
        if ($application->status !== $containerStatus) {
            $application->status = $containerStatus;
            $application->save();
-        } else {
-            $application->update(['last_online_at' => now()]);
        }
    }

@@ -479,9 +638,7 @@ class PushServerUpdateJob implements ShouldBeEncrypted, ShouldQueue, Silenced
            $applicationId = $parts[0];
            $pullRequestId = $parts[1];

-            $applicationPreview = $this->previews->where('application_id', $applicationId)
-                ->where('pull_request_id', $pullRequestId)
-                ->first();
+            $applicationPreview = $this->previewsByKey->get($applicationId.':'.$pullRequestId);

            if ($applicationPreview && ! str($applicationPreview->status)->startsWith('exited')) {
                $previewIdsToUpdate->push($applicationPreview->id);
@@ -520,15 +677,13 @@ class PushServerUpdateJob implements ShouldBeEncrypted, ShouldQueue, Silenced

    private function updateDatabaseStatus(string $databaseUuid, string $containerStatus, bool $tcpProxy = false)
    {
-        $database = $this->databases->where('uuid', $databaseUuid)->first();
+        $database = $this->databasesByUuid->get($databaseUuid);
        if (! $database) {
            return;
        }
        if ($database->status !== $containerStatus) {
            $database->status = $containerStatus;
            $database->save();
-        } else {
-            $database->update(['last_online_at' => now()]);
        }
        if ($this->isRunning($containerStatus) && $tcpProxy) {
            $tcpProxyContainerFound = $this->containers->filter(function ($value, $key) use ($databaseUuid) {
@@ -563,7 +718,7 @@ class PushServerUpdateJob implements ShouldBeEncrypted, ShouldQueue, Silenced
        }

        $notFoundDatabaseUuids->each(function ($databaseUuid) {
-            $database = $this->databases->where('uuid', $databaseUuid)->first();
+            $database = $this->databasesByUuid->get($databaseUuid);
            if ($database) {
                if (! str($database->status)->startsWith('exited')) {
                    $database->update([