tiennm99/coolify
1
0 Fork 0
mirror of https://github.com/tiennm99/coolify.git synced 2026-10-17 04:15:02 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix: correct status for excluded health check containers (#7283)

Browse Source
This commit is contained in:
Andras Bacsai authored and GitHub committed 2025-11-21 09:17:26 +01:00
2 parents 9aff301442 01609e7f8b
commit 355dcc186c
32 files changed
+3720 -344
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

app/Actions/Docker/GetContainersStatus.php
+104 -69
View File
@@ -8,6 +8,8 @@ use App\Events\ServiceChecked;
use App\Models\ApplicationPreview;
use App\Models\Server;
use App\Models\ServiceDatabase;
use App\Services\ContainerStatusAggregator;
use App\Traits\CalculatesExcludedStatus;
use Illuminate\Support\Arr;
use Illuminate\Support\Collection;
use Illuminate\Support\Facades\DB;
@@ -16,6 +18,7 @@ use Lorisleiva\Actions\Concerns\AsAction;
class GetContainersStatus
{
use AsAction;
use CalculatesExcludedStatus;
public string $jobQueue = 'high';
@@ -31,6 +34,8 @@ class GetContainersStatus
protected ?Collection $applicationContainerRestartCounts;
protected ?Collection $serviceContainerStatuses;
public function handle(Server $server, ?Collection $containers = null, ?Collection $containerReplicates = null)
{
$this->containers = $containers;
@@ -98,11 +103,13 @@ class GetContainersStatus
$labels = data_get($container, 'Config.Labels');
}
$containerStatus = data_get($container, 'State.Status');
$containerHealth = data_get($container, 'State.Health.Status', 'unhealthy');
$containerHealth = data_get($container, 'State.Health.Status');
if ($containerStatus === 'restarting') {
$containerStatus = "restarting ($containerHealth)";
$healthSuffix = $containerHealth ?? 'unknown';
$containerStatus = "restarting:$healthSuffix";
} else {
$containerStatus = "$containerStatus ($containerHealth)";
$healthSuffix = $containerHealth ?? 'unknown';
$containerStatus = "$containerStatus:$healthSuffix";
}
$labels = Arr::undot(format_docker_labels_to_json($labels));
$applicationId = data_get($labels, 'coolify.applicationId');
@@ -222,23 +229,34 @@ class GetContainersStatus
if ($serviceLabelId) {
$subType = data_get($labels, 'coolify.service.subType');
$subId = data_get($labels, 'coolify.service.subId');
$service = $services->where('id', $serviceLabelId)->first();
if (! $service) {
$parentService = $services->where('id', $serviceLabelId)->first();
if (! $parentService) {
continue;
}
// Store container status for aggregation
if (! isset($this->serviceContainerStatuses)) {
$this->serviceContainerStatuses = collect();
}
$key = $serviceLabelId.':'.$subType.':'.$subId;
if (! $this->serviceContainerStatuses->has($key)) {
$this->serviceContainerStatuses->put($key, collect());
}
$containerName = data_get($labels, 'com.docker.compose.service');
if ($containerName) {
$this->serviceContainerStatuses->get($key)->put($containerName, $containerStatus);
}
// Mark service as found
if ($subType === 'application') {
$service = $service->applications()->where('id', $subId)->first();
$service = $parentService->applications()->where('id', $subId)->first();
} else {
$service = $service->databases()->where('id', $subId)->first();
$service = $parentService->databases()->where('id', $subId)->first();
}
if ($service) {
$foundServices[] = "$service->id-$service->name";
$statusFromDb = $service->status;
if ($statusFromDb !== $containerStatus) {
$service->update(['status' => $containerStatus]);
} else {
$service->update(['last_online_at' => now()]);
}
}
}
}
@@ -418,6 +436,9 @@ class GetContainersStatus
}
}
// Aggregate multi-container service statuses
$this->aggregateServiceContainerStatuses($services);
ServiceChecked::dispatch($this->server->team->id);
}
@@ -425,74 +446,88 @@ class GetContainersStatus
{
// Parse docker compose to check for excluded containers
$dockerComposeRaw = data_get($application, 'docker_compose_raw');
$excludedContainers = collect();
if ($dockerComposeRaw) {
try {
$dockerCompose = \Symfony\Component\Yaml\Yaml::parse($dockerComposeRaw);
$services = data_get($dockerCompose, 'services', []);
foreach ($services as $serviceName => $serviceConfig) {
// Check if container should be excluded
$excludeFromHc = data_get($serviceConfig, 'exclude_from_hc', false);
$restartPolicy = data_get($serviceConfig, 'restart', 'always');
if ($excludeFromHc || $restartPolicy === 'no') {
$excludedContainers->push($serviceName);
}
}
} catch (\Exception $e) {
// If we can't parse, treat all containers as included
}
}
$excludedContainers = $this->getExcludedContainersFromDockerCompose($dockerComposeRaw);
// Filter out excluded containers
$relevantStatuses = $containerStatuses->filter(function ($status, $containerName) use ($excludedContainers) {
return ! $excludedContainers->contains($containerName);
});
// If all containers are excluded, don't update status
// If all containers are excluded, calculate status from excluded containers
if ($relevantStatuses->isEmpty()) {
return null;
return $this->calculateExcludedStatusFromStrings($containerStatuses);
}
$hasRunning = false;
$hasRestarting = false;
$hasUnhealthy = false;
$hasExited = false;
// Use ContainerStatusAggregator service for state machine logic
$aggregator = new ContainerStatusAggregator;
foreach ($relevantStatuses as $status) {
if (str($status)->contains('restarting')) {
$hasRestarting = true;
} elseif (str($status)->contains('running')) {
$hasRunning = true;
if (str($status)->contains('unhealthy')) {
$hasUnhealthy = true;
return $aggregator->aggregateFromStrings($relevantStatuses, $maxRestartCount);
}
private function aggregateServiceContainerStatuses($services)
{
if (! isset($this->serviceContainerStatuses) || $this->serviceContainerStatuses->isEmpty()) {
return;
}
foreach ($this->serviceContainerStatuses as $key => $containerStatuses) {
// Parse key: serviceId:subType:subId
[$serviceId, $subType, $subId] = explode(':', $key);
$service = $services->where('id', $serviceId)->first();
if (! $service) {
continue;
}
// Get the service sub-resource (ServiceApplication or ServiceDatabase)
$subResource = null;
if ($subType === 'application') {
$subResource = $service->applications()->where('id', $subId)->first();
} elseif ($subType === 'database') {
$subResource = $service->databases()->where('id', $subId)->first();
}
if (! $subResource) {
continue;
}
// Parse docker compose from service to check for excluded containers
$dockerComposeRaw = data_get($service, 'docker_compose_raw');
$excludedContainers = $this->getExcludedContainersFromDockerCompose($dockerComposeRaw);
// Filter out excluded containers
$relevantStatuses = $containerStatuses->filter(function ($status, $containerName) use ($excludedContainers) {
return ! $excludedContainers->contains($containerName);
});
// If all containers are excluded, calculate status from excluded containers
if ($relevantStatuses->isEmpty()) {
$aggregatedStatus = $this->calculateExcludedStatusFromStrings($containerStatuses);
if ($aggregatedStatus) {
$statusFromDb = $subResource->status;
if ($statusFromDb !== $aggregatedStatus) {
$subResource->update(['status' => $aggregatedStatus]);
} else {
$subResource->update(['last_online_at' => now()]);
}
}
continue;
}
// Use ContainerStatusAggregator service for state machine logic
$aggregator = new ContainerStatusAggregator;
$aggregatedStatus = $aggregator->aggregateFromStrings($relevantStatuses);
// Update service sub-resource status with aggregated result
if ($aggregatedStatus) {
$statusFromDb = $subResource->status;
if ($statusFromDb !== $aggregatedStatus) {
$subResource->update(['status' => $aggregatedStatus]);
} else {
$subResource->update(['last_online_at' => now()]);
}
} elseif (str($status)->contains('exited')) {
$hasExited = true;
$hasUnhealthy = true;
}
}
if ($hasRestarting) {
return 'degraded (unhealthy)';
}
// If container is exited but has restart count > 0, it's in a crash loop
if ($hasExited && $maxRestartCount > 0) {
return 'degraded (unhealthy)';
}
if ($hasRunning && $hasExited) {
return 'degraded (unhealthy)';
}
if ($hasRunning) {
return $hasUnhealthy ? 'running (unhealthy)' : 'running (healthy)';
}
// All containers are exited with no restart count - truly stopped
return 'exited (unhealthy)';
}
}
app/Actions/Shared/ComplexStatusCheck.php
+15 -61
View File
@@ -3,11 +3,14 @@
namespace App\Actions\Shared;
use App\Models\Application;
use App\Services\ContainerStatusAggregator;
use App\Traits\CalculatesExcludedStatus;
use Lorisleiva\Actions\Concerns\AsAction;
class ComplexStatusCheck
{
use AsAction;
use CalculatesExcludedStatus;
public function handle(Application $application)
{
@@ -61,74 +64,25 @@ class ComplexStatusCheck
private function aggregateContainerStatuses($application, $containers)
{
$dockerComposeRaw = data_get($application, 'docker_compose_raw');
$excludedContainers = collect();
$excludedContainers = $this->getExcludedContainersFromDockerCompose($dockerComposeRaw);
if ($dockerComposeRaw) {
try {
$dockerCompose = \Symfony\Component\Yaml\Yaml::parse($dockerComposeRaw);
$services = data_get($dockerCompose, 'services', []);
foreach ($services as $serviceName => $serviceConfig) {
$excludeFromHc = data_get($serviceConfig, 'exclude_from_hc', false);
$restartPolicy = data_get($serviceConfig, 'restart', 'always');
if ($excludeFromHc || $restartPolicy === 'no') {
$excludedContainers->push($serviceName);
}
}
} catch (\Exception $e) {
// If we can't parse, treat all containers as included
}
}
$hasRunning = false;
$hasRestarting = false;
$hasUnhealthy = false;
$hasExited = false;
$relevantContainerCount = 0;
foreach ($containers as $container) {
// Filter non-excluded containers
$relevantContainers = collect($containers)->filter(function ($container) use ($excludedContainers) {
$labels = data_get($container, 'Config.Labels', []);
$serviceName = data_get($labels, 'com.docker.compose.service');
if ($serviceName && $excludedContainers->contains($serviceName)) {
continue;
}
return ! ($serviceName && $excludedContainers->contains($serviceName));
});
$relevantContainerCount++;
$containerStatus = data_get($container, 'State.Status');
$containerHealth = data_get($container, 'State.Health.Status', 'unhealthy');
if ($containerStatus === 'restarting') {
$hasRestarting = true;
$hasUnhealthy = true;
} elseif ($containerStatus === 'running') {
$hasRunning = true;
if ($containerHealth === 'unhealthy') {
$hasUnhealthy = true;
}
} elseif ($containerStatus === 'exited') {
$hasExited = true;
$hasUnhealthy = true;
}
// If all containers are excluded, calculate status from excluded containers
// but mark it with :excluded to indicate monitoring is disabled
if ($relevantContainers->isEmpty()) {
return $this->calculateExcludedStatus($containers, $excludedContainers);
}
if ($relevantContainerCount === 0) {
return 'running:healthy';
}
// Use ContainerStatusAggregator service for state machine logic
$aggregator = new ContainerStatusAggregator;
if ($hasRestarting) {
return 'degraded:unhealthy';
}
if ($hasRunning && $hasExited) {
return 'degraded:unhealthy';
}
if ($hasRunning) {
return $hasUnhealthy ? 'running:unhealthy' : 'running:healthy';
}
return 'exited:unhealthy';
return $aggregator->aggregateFromContainers($relevantContainers);
}
}
app/Jobs/PushServerUpdateJob.php
+156 -97
View File
@@ -13,6 +13,8 @@ use App\Models\Server;
use App\Models\ServiceApplication;
use App\Models\ServiceDatabase;
use App\Notifications\Container\ContainerRestarted;
use App\Services\ContainerStatusAggregator;
use App\Traits\CalculatesExcludedStatus;
use Illuminate\Bus\Queueable;
use Illuminate\Contracts\Queue\ShouldBeEncrypted;
use Illuminate\Contracts\Queue\ShouldQueue;
@@ -25,6 +27,7 @@ use Laravel\Horizon\Contracts\Silenced;
class PushServerUpdateJob implements ShouldBeEncrypted, ShouldQueue, Silenced
{
use CalculatesExcludedStatus;
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
public $tries = 1;
@@ -67,6 +70,8 @@ class PushServerUpdateJob implements ShouldBeEncrypted, ShouldQueue, Silenced
public Collection $applicationContainerStatuses;
public Collection $serviceContainerStatuses;
public bool $foundProxy = false;
public bool $foundLogDrainContainer = false;
@@ -90,6 +95,7 @@ class PushServerUpdateJob implements ShouldBeEncrypted, ShouldQueue, Silenced
$this->foundApplicationPreviewsIds = collect();
$this->foundServiceDatabaseIds = collect();
$this->applicationContainerStatuses = collect();
$this->serviceContainerStatuses = collect();
$this->allApplicationIds = collect();
$this->allDatabaseUuids = collect();
$this->allTcpProxyUuids = collect();
@@ -108,7 +114,6 @@ class PushServerUpdateJob implements ShouldBeEncrypted, ShouldQueue, Silenced
$this->server->sentinelHeartbeat();
$this->containers = collect(data_get($data, 'containers'));
$filesystemUsageRoot = data_get($data, 'filesystem_usage_root.used_percentage');
ServerStorageCheckJob::dispatch($this->server, $filesystemUsageRoot);
@@ -141,65 +146,85 @@ class PushServerUpdateJob implements ShouldBeEncrypted, ShouldQueue, Silenced
foreach ($this->containers as $container) {
$containerStatus = data_get($container, 'state', 'exited');
$containerHealth = data_get($container, 'health_status', 'unhealthy');
$containerStatus = "$containerStatus ($containerHealth)";
$rawHealthStatus = data_get($container, 'health_status');
$containerHealth = $rawHealthStatus ?? 'unknown';
$containerStatus = "$containerStatus:$containerHealth";
$labels = collect(data_get($container, 'labels'));
$coolify_managed = $labels->has('coolify.managed');
if ($coolify_managed) {
$name = data_get($container, 'name');
if ($name === 'coolify-log-drain' && $this->isRunning($containerStatus)) {
$this->foundLogDrainContainer = true;
}
if ($labels->has('coolify.applicationId')) {
$applicationId = $labels->get('coolify.applicationId');
$pullRequestId = $labels->get('coolify.pullRequestId', '0');
try {
if ($pullRequestId === '0') {
if ($this->allApplicationIds->contains($applicationId) && $this->isRunning($containerStatus)) {
$this->foundApplicationIds->push($applicationId);
}
// Store container status for aggregation
if (! $this->applicationContainerStatuses->has($applicationId)) {
$this->applicationContainerStatuses->put($applicationId, collect());
}
$containerName = $labels->get('com.docker.compose.service');
if ($containerName) {
$this->applicationContainerStatuses->get($applicationId)->put($containerName, $containerStatus);
}
} else {
$previewKey = $applicationId.':'.$pullRequestId;
if ($this->allApplicationPreviewsIds->contains($previewKey) && $this->isRunning($containerStatus)) {
$this->foundApplicationPreviewsIds->push($previewKey);
}
$this->updateApplicationPreviewStatus($applicationId, $pullRequestId, $containerStatus);
if (! $coolify_managed) {
continue;
}
$name = data_get($container, 'name');
if ($name === 'coolify-log-drain' && $this->isRunning($containerStatus)) {
$this->foundLogDrainContainer = true;
}
if ($labels->has('coolify.applicationId')) {
$applicationId = $labels->get('coolify.applicationId');
$pullRequestId = $labels->get('coolify.pullRequestId', '0');
try {
if ($pullRequestId === '0') {
if ($this->allApplicationIds->contains($applicationId)) {
$this->foundApplicationIds->push($applicationId);
}
// Store container status for aggregation
if (! $this->applicationContainerStatuses->has($applicationId)) {
$this->applicationContainerStatuses->put($applicationId, collect());
}
$containerName = $labels->get('com.docker.compose.service');
if ($containerName) {
$this->applicationContainerStatuses->get($applicationId)->put($containerName, $containerStatus);
}
} catch (\Exception $e) {
}
} elseif ($labels->has('coolify.serviceId')) {
$serviceId = $labels->get('coolify.serviceId');
$subType = $labels->get('coolify.service.subType');
$subId = $labels->get('coolify.service.subId');
if ($subType === 'application' && $this->isRunning($containerStatus)) {
$this->foundServiceApplicationIds->push($subId);
$this->updateServiceSubStatus($serviceId, $subType, $subId, $containerStatus);
} elseif ($subType === 'database' && $this->isRunning($containerStatus)) {
$this->foundServiceDatabaseIds->push($subId);
$this->updateServiceSubStatus($serviceId, $subType, $subId, $containerStatus);
}
} else {
$uuid = $labels->get('com.docker.compose.service');
$type = $labels->get('coolify.type');
if ($name === 'coolify-proxy' && $this->isRunning($containerStatus)) {
$this->foundProxy = true;
} elseif ($type === 'service' && $this->isRunning($containerStatus)) {
} else {
if ($this->allDatabaseUuids->contains($uuid) && $this->isRunning($containerStatus)) {
$this->foundDatabaseUuids->push($uuid);
if ($this->allTcpProxyUuids->contains($uuid) && $this->isRunning($containerStatus)) {
$this->updateDatabaseStatus($uuid, $containerStatus, tcpProxy: true);
} else {
$this->updateDatabaseStatus($uuid, $containerStatus, tcpProxy: false);
}
$previewKey = $applicationId.':'.$pullRequestId;
if ($this->allApplicationPreviewsIds->contains($previewKey)) {
$this->foundApplicationPreviewsIds->push($previewKey);
}
$this->updateApplicationPreviewStatus($applicationId, $pullRequestId, $containerStatus);
}
} catch (\Exception $e) {
}
} elseif ($labels->has('coolify.serviceId')) {
$serviceId = $labels->get('coolify.serviceId');
$subType = $labels->get('coolify.service.subType');
$subId = $labels->get('coolify.service.subId');
if ($subType === 'application') {
$this->foundServiceApplicationIds->push($subId);
// Store container status for aggregation
$key = $serviceId.':'.$subType.':'.$subId;
if (! $this->serviceContainerStatuses->has($key)) {
$this->serviceContainerStatuses->put($key, collect());
}
$containerName = $labels->get('com.docker.compose.service');
if ($containerName) {
$this->serviceContainerStatuses->get($key)->put($containerName, $containerStatus);
}
} elseif ($subType === 'database') {
$this->foundServiceDatabaseIds->push($subId);
// Store container status for aggregation
$key = $serviceId.':'.$subType.':'.$subId;
if (! $this->serviceContainerStatuses->has($key)) {
$this->serviceContainerStatuses->put($key, collect());
}
$containerName = $labels->get('com.docker.compose.service');
if ($containerName) {
$this->serviceContainerStatuses->get($key)->put($containerName, $containerStatus);
}
}
} else {
$uuid = $labels->get('com.docker.compose.service');
$type = $labels->get('coolify.type');
if ($name === 'coolify-proxy' && $this->isRunning($containerStatus)) {
$this->foundProxy = true;
} elseif ($type === 'service' && $this->isRunning($containerStatus)) {
} else {
if ($this->allDatabaseUuids->contains($uuid) && $this->isRunning($containerStatus)) {
$this->foundDatabaseUuids->push($uuid);
if ($this->allTcpProxyUuids->contains($uuid) && $this->isRunning($containerStatus)) {
$this->updateDatabaseStatus($uuid, $containerStatus, tcpProxy: true);
} else {
$this->updateDatabaseStatus($uuid, $containerStatus, tcpProxy: false);
}
}
}
@@ -218,6 +243,9 @@ class PushServerUpdateJob implements ShouldBeEncrypted, ShouldQueue, Silenced
// Aggregate multi-container application statuses
$this->aggregateMultiContainerStatuses();
// Aggregate multi-container service statuses
$this->aggregateServiceContainerStatuses();
$this->checkLogDrainContainer();
}
@@ -235,57 +263,28 @@ class PushServerUpdateJob implements ShouldBeEncrypted, ShouldQueue, Silenced
// Parse docker compose to check for excluded containers
$dockerComposeRaw = data_get($application, 'docker_compose_raw');
$excludedContainers = collect();
if ($dockerComposeRaw) {
try {
$dockerCompose = \Symfony\Component\Yaml\Yaml::parse($dockerComposeRaw);
$services = data_get($dockerCompose, 'services', []);
foreach ($services as $serviceName => $serviceConfig) {
// Check if container should be excluded
$excludeFromHc = data_get($serviceConfig, 'exclude_from_hc', false);
$restartPolicy = data_get($serviceConfig, 'restart', 'always');
if ($excludeFromHc || $restartPolicy === 'no') {
$excludedContainers->push($serviceName);
}
}
} catch (\Exception $e) {
// If we can't parse, treat all containers as included
}
}
$excludedContainers = $this->getExcludedContainersFromDockerCompose($dockerComposeRaw);
// Filter out excluded containers
$relevantStatuses = $containerStatuses->filter(function ($status, $containerName) use ($excludedContainers) {
return ! $excludedContainers->contains($containerName);
});
// If all containers are excluded, don't update status
// If all containers are excluded, calculate status from excluded containers
if ($relevantStatuses->isEmpty()) {
$aggregatedStatus = $this->calculateExcludedStatusFromStrings($containerStatuses);
if ($aggregatedStatus && $application->status !== $aggregatedStatus) {
$application->status = $aggregatedStatus;
$application->save();
}
continue;
}
// Aggregate status: if any container is running, app is running
$hasRunning = false;
$hasUnhealthy = false;
foreach ($relevantStatuses as $status) {
if (str($status)->contains('running')) {
$hasRunning = true;
if (str($status)->contains('unhealthy')) {
$hasUnhealthy = true;
}
}
}
$aggregatedStatus = null;
if ($hasRunning) {
$aggregatedStatus = $hasUnhealthy ? 'running (unhealthy)' : 'running (healthy)';
} else {
// All containers are exited
$aggregatedStatus = 'exited (unhealthy)';
}
// Use ContainerStatusAggregator service for state machine logic
$aggregator = new ContainerStatusAggregator;
$aggregatedStatus = $aggregator->aggregateFromStrings($relevantStatuses, 0);
// Update application status with aggregated result
if ($aggregatedStatus && $application->status !== $aggregatedStatus) {
@@ -295,6 +294,66 @@ class PushServerUpdateJob implements ShouldBeEncrypted, ShouldQueue, Silenced
}
}
private function aggregateServiceContainerStatuses()
{
if ($this->serviceContainerStatuses->isEmpty()) {
return;
}
foreach ($this->serviceContainerStatuses as $key => $containerStatuses) {
// Parse key: serviceId:subType:subId
[$serviceId, $subType, $subId] = explode(':', $key);
$service = $this->services->where('id', $serviceId)->first();
if (! $service) {
continue;
}
// Get the service sub-resource (ServiceApplication or ServiceDatabase)
$subResource = null;
if ($subType === 'application') {
$subResource = $service->applications()->where('id', $subId)->first();
} elseif ($subType === 'database') {
$subResource = $service->databases()->where('id', $subId)->first();
}
if (! $subResource) {
continue;
}
// Parse docker compose from service to check for excluded containers
$dockerComposeRaw = data_get($service, 'docker_compose_raw');
$excludedContainers = $this->getExcludedContainersFromDockerCompose($dockerComposeRaw);
// Filter out excluded containers
$relevantStatuses = $containerStatuses->filter(function ($status, $containerName) use ($excludedContainers) {
return ! $excludedContainers->contains($containerName);
});
// If all containers are excluded, calculate status from excluded containers
if ($relevantStatuses->isEmpty()) {
$aggregatedStatus = $this->calculateExcludedStatusFromStrings($containerStatuses);
if ($aggregatedStatus && $subResource->status !== $aggregatedStatus) {
$subResource->status = $aggregatedStatus;
$subResource->save();
}
continue;
}
// Use ContainerStatusAggregator service for state machine logic
// NOTE: Sentinel does NOT provide restart count data, so maxRestartCount is always 0
$aggregator = new ContainerStatusAggregator;
$aggregatedStatus = $aggregator->aggregateFromStrings($relevantStatuses, 0);
// Update service sub-resource status with aggregated result
if ($aggregatedStatus && $subResource->status !== $aggregatedStatus) {
$subResource->status = $aggregatedStatus;
$subResource->save();
}
}
}
private function updateApplicationStatus(string $applicationId, string $containerStatus)
{
$application = $this->applications->where('id', $applicationId)->first();
app/Models/Application.php
+12 -10
View File
@@ -669,21 +669,23 @@ class Application extends BaseModel
{
return Attribute::make(
get: function () {
if (! $this->relationLoaded('additional_servers') || $this->additional_servers->count() === 0) {
return $this->destination?->server?->isFunctional() ?? false;
// Check main server infrastructure health
$main_server_functional = $this->destination?->server?->isFunctional() ?? false;
if (! $main_server_functional) {
return false;
}
$additional_servers_status = $this->additional_servers->pluck('pivot.status');
$main_server_status = $this->destination?->server?->isFunctional() ?? false;
foreach ($additional_servers_status as $status) {
$server_status = str($status)->before(':')->value();
if ($server_status !== 'running') {
return false;
// Check additional servers infrastructure health (not container status!)
if ($this->relationLoaded('additional_servers') && $this->additional_servers->count() > 0) {
foreach ($this->additional_servers as $server) {
if (! $server->isFunctional()) {
return false; // Real server infrastructure problem
}
}
}
return $main_server_status;
return true;
}
);
}
app/Models/Service.php
+99 -57
View File
@@ -3,6 +3,7 @@
namespace App\Models;
use App\Enums\ProcessStatus;
use App\Services\ContainerStatusAggregator;
use App\Traits\ClearsGlobalSearchCache;
use App\Traits\HasSafeStringAttribute;
use Illuminate\Database\Eloquent\Casts\Attribute;
@@ -173,6 +174,21 @@ class Service extends BaseModel
instant_remote_process(["docker network rm {$this->uuid}"], $server, false);
}
/**
* Calculate the service's aggregate status from its applications and databases.
*
* This method aggregates status from Eloquent model relationships (not Docker containers).
* It differs from the CalculatesExcludedStatus trait which works with Docker container objects
* during container inspection. This accessor runs on-demand for UI display and works with
* already-stored status strings from the database.
*
* Status format: "{status}:{health}" or "{status}:{health}:excluded"
* - Status values: running, exited, degraded, starting, paused, restarting
* - Health values: healthy, unhealthy, unknown
* - :excluded suffix: Indicates all containers are excluded from health monitoring
*
* @return string The aggregate status in format "status:health" or "status:health:excluded"
*/
public function getStatusAttribute()
{
if ($this->isStarting()) {
@@ -182,71 +198,97 @@ class Service extends BaseModel
$applications = $this->applications;
$databases = $this->databases;
$complexStatus = null;
$complexHealth = null;
[$complexStatus, $complexHealth, $hasNonExcluded] = $this->aggregateResourceStatuses(
$applications,
$databases,
excludedOnly: false
);
foreach ($applications as $application) {
if ($application->exclude_from_status) {
continue;
// If all services are excluded from status checks, calculate status from excluded containers
// but mark it with :excluded to indicate monitoring is disabled
if (! $hasNonExcluded && ($complexStatus === null && $complexHealth === null)) {
[$excludedStatus, $excludedHealth] = $this->aggregateResourceStatuses(
$applications,
$databases,
excludedOnly: true
);
// Return status with :excluded suffix to indicate monitoring is disabled
if ($excludedStatus && $excludedHealth) {
return "{$excludedStatus}:{$excludedHealth}:excluded";
}
$status = str($application->status)->before('(')->trim();
$health = str($application->status)->between('(', ')')->trim();
if ($complexStatus === 'degraded') {
continue;
}
if ($status->startsWith('running')) {
if ($complexStatus === 'exited') {
$complexStatus = 'degraded';
} else {
$complexStatus = 'running';
}
} elseif ($status->startsWith('restarting')) {
$complexStatus = 'degraded';
} elseif ($status->startsWith('exited')) {
$complexStatus = 'exited';
}
if ($health->value() === 'healthy') {
if ($complexHealth === 'unhealthy') {
continue;
}
$complexHealth = 'healthy';
} else {
$complexHealth = 'unhealthy';
}
}
foreach ($databases as $database) {
if ($database->exclude_from_status) {
continue;
}
$status = str($database->status)->before('(')->trim();
$health = str($database->status)->between('(', ')')->trim();
if ($complexStatus === 'degraded') {
continue;
}
if ($status->startsWith('running')) {
if ($complexStatus === 'exited') {
$complexStatus = 'degraded';
} else {
$complexStatus = 'running';
}
} elseif ($status->startsWith('restarting')) {
$complexStatus = 'degraded';
} elseif ($status->startsWith('exited')) {
$complexStatus = 'exited';
}
if ($health->value() === 'healthy') {
if ($complexHealth === 'unhealthy') {
continue;
}
$complexHealth = 'healthy';
} else {
$complexHealth = 'unhealthy';
// If no status was calculated at all (no containers exist), return unknown
if ($excludedStatus === null && $excludedHealth === null) {
return 'unknown:unknown:excluded';
}
return 'exited:unhealthy:excluded';
}
return "{$complexStatus}:{$complexHealth}";
}
/**
* Aggregate status and health from collections of applications and databases.
*
* This helper method consolidates status aggregation logic using ContainerStatusAggregator.
* It processes container status strings stored in the database (not live Docker data).
*
* @param \Illuminate\Database\Eloquent\Collection $applications Collection of Application models
* @param \Illuminate\Database\Eloquent\Collection $databases Collection of Database models
* @param bool $excludedOnly If true, only process excluded containers; if false, only process non-excluded
* @return array{0: string|null, 1: string|null, 2?: bool} [status, health, hasNonExcluded (only when excludedOnly=false)]
*/
private function aggregateResourceStatuses($applications, $databases, bool $excludedOnly = false): array
{
$hasNonExcluded = false;
$statusStrings = collect();
// Process both applications and databases using the same logic
$resources = $applications->concat($databases);
foreach ($resources as $resource) {
$isExcluded = $resource->exclude_from_status || str($resource->status)->contains(':excluded');
// Filter based on excludedOnly flag
if ($excludedOnly && ! $isExcluded) {
continue;
}
if (! $excludedOnly && $isExcluded) {
continue;
}
if (! $excludedOnly) {
$hasNonExcluded = true;
}
// Strip :excluded suffix before aggregation (it's in the 3rd part of "status:health:excluded")
$status = str($resource->status)->before(':excluded')->toString();
$statusStrings->push($status);
}
// If no status strings collected, return nulls
if ($statusStrings->isEmpty()) {
return $excludedOnly ? [null, null] : [null, null, $hasNonExcluded];
}
// Use ContainerStatusAggregator service for state machine logic
$aggregator = new ContainerStatusAggregator;
$aggregatedStatus = $aggregator->aggregateFromStrings($statusStrings);
// Parse the aggregated "status:health" string
$parts = explode(':', $aggregatedStatus);
$status = $parts[0] ?? null;
$health = $parts[1] ?? null;
if ($excludedOnly) {
return [$status, $health];
}
return [$status, $health, $hasNonExcluded];
}
public function extraFields()
{
$fields = collect([]);
app/Services/ContainerStatusAggregator.php
+253
View File
@@ -0,0 +1,253 @@
<?php
namespace App\Services;
use Illuminate\Support\Collection;
use Illuminate\Support\Facades\Log;
/**
* Container Status Aggregator Service
*
* Centralized service for aggregating container statuses into a single status string.
* Uses a priority-based state machine to determine the overall status from multiple containers.
*
* Output Format: Colon-separated (e.g., "running:healthy", "degraded:unhealthy")
* This format is used throughout the backend for consistency and machine-readability.
* UI components transform this to human-readable format (e.g., "Running (Healthy)").
*
* State Priority (highest to lowest):
* 1. Restarting → degraded:unhealthy
* 2. Crash Loop (exited with restarts) → degraded:unhealthy
* 3. Mixed (running + exited) → degraded:unhealthy
* 4. Running → running:healthy/unhealthy/unknown
* 5. Dead/Removing → degraded:unhealthy
* 6. Paused → paused:unknown
* 7. Starting/Created → starting:unknown
* 8. Exited → exited:unhealthy
*/
class ContainerStatusAggregator
{
/**
* Aggregate container statuses from status strings into a single status.
*
* @param Collection $containerStatuses Collection of status strings (e.g., "running (healthy)", "running:healthy")
* @param int $maxRestartCount Maximum restart count across containers (for crash loop detection)
* @return string Aggregated status in colon format (e.g., "running:healthy")
*/
public function aggregateFromStrings(Collection $containerStatuses, int $maxRestartCount = 0): string
{
// Validate maxRestartCount parameter
if ($maxRestartCount < 0) {
Log::warning('Negative maxRestartCount corrected to 0', [
'original_value' => $maxRestartCount,
]);
$maxRestartCount = 0;
}
if ($maxRestartCount > 1000) {
Log::warning('High maxRestartCount detected', [
'maxRestartCount' => $maxRestartCount,
'containers' => $containerStatuses->count(),
]);
}
if ($containerStatuses->isEmpty()) {
return 'exited:unhealthy';
}
// Initialize state flags
$hasRunning = false;
$hasRestarting = false;
$hasUnhealthy = false;
$hasUnknown = false;
$hasExited = false;
$hasStarting = false;
$hasPaused = false;
$hasDead = false;
// Parse each status string and set flags
foreach ($containerStatuses as $status) {
if (str($status)->contains('restarting')) {
$hasRestarting = true;
} elseif (str($status)->contains('running')) {
$hasRunning = true;
if (str($status)->contains('unhealthy')) {
$hasUnhealthy = true;
}
if (str($status)->contains('unknown')) {
$hasUnknown = true;
}
} elseif (str($status)->contains('exited')) {
$hasExited = true;
$hasUnhealthy = true;
} elseif (str($status)->contains('created') || str($status)->contains('starting')) {
$hasStarting = true;
} elseif (str($status)->contains('paused')) {
$hasPaused = true;
} elseif (str($status)->contains('dead') || str($status)->contains('removing')) {
$hasDead = true;
}
}
// Priority-based status resolution
return $this->resolveStatus(
$hasRunning,
$hasRestarting,
$hasUnhealthy,
$hasUnknown,
$hasExited,
$hasStarting,
$hasPaused,
$hasDead,
$maxRestartCount
);
}
/**
* Aggregate container statuses from Docker container objects.
*
* @param Collection $containers Collection of Docker container objects with State property
* @param int $maxRestartCount Maximum restart count across containers (for crash loop detection)
* @return string Aggregated status in colon format (e.g., "running:healthy")
*/
public function aggregateFromContainers(Collection $containers, int $maxRestartCount = 0): string
{
// Validate maxRestartCount parameter
if ($maxRestartCount < 0) {
Log::warning('Negative maxRestartCount corrected to 0', [
'original_value' => $maxRestartCount,
]);
$maxRestartCount = 0;
}
if ($maxRestartCount > 1000) {
Log::warning('High maxRestartCount detected', [
'maxRestartCount' => $maxRestartCount,
'containers' => $containers->count(),
]);
}
if ($containers->isEmpty()) {
return 'exited:unhealthy';
}
// Initialize state flags
$hasRunning = false;
$hasRestarting = false;
$hasUnhealthy = false;
$hasUnknown = false;
$hasExited = false;
$hasStarting = false;
$hasPaused = false;
$hasDead = false;
// Parse each container object and set flags
foreach ($containers as $container) {
$state = data_get($container, 'State.Status', 'exited');
$health = data_get($container, 'State.Health.Status');
if ($state === 'restarting') {
$hasRestarting = true;
} elseif ($state === 'running') {
$hasRunning = true;
if ($health === 'unhealthy') {
$hasUnhealthy = true;
} elseif (is_null($health) || $health === 'starting') {
$hasUnknown = true;
}
} elseif ($state === 'exited') {
$hasExited = true;
$hasUnhealthy = true;
} elseif ($state === 'created' || $state === 'starting') {
$hasStarting = true;
} elseif ($state === 'paused') {
$hasPaused = true;
} elseif ($state === 'dead' || $state === 'removing') {
$hasDead = true;
}
}
// Priority-based status resolution
return $this->resolveStatus(
$hasRunning,
$hasRestarting,
$hasUnhealthy,
$hasUnknown,
$hasExited,
$hasStarting,
$hasPaused,
$hasDead,
$maxRestartCount
);
}
/**
* Resolve the aggregated status based on state flags (priority-based state machine).
*
* @param bool $hasRunning Has at least one running container
* @param bool $hasRestarting Has at least one restarting container
* @param bool $hasUnhealthy Has at least one unhealthy container
* @param bool $hasUnknown Has at least one container with unknown health
* @param bool $hasExited Has at least one exited container
* @param bool $hasStarting Has at least one starting/created container
* @param bool $hasPaused Has at least one paused container
* @param bool $hasDead Has at least one dead/removing container
* @param int $maxRestartCount Maximum restart count (for crash loop detection)
* @return string Status in colon format (e.g., "running:healthy")
*/
private function resolveStatus(
bool $hasRunning,
bool $hasRestarting,
bool $hasUnhealthy,
bool $hasUnknown,
bool $hasExited,
bool $hasStarting,
bool $hasPaused,
bool $hasDead,
int $maxRestartCount
): string {
// Priority 1: Restarting containers (degraded state)
if ($hasRestarting) {
return 'degraded:unhealthy';
}
// Priority 2: Crash loop detection (exited with restart count > 0)
if ($hasExited && $maxRestartCount > 0) {
return 'degraded:unhealthy';
}
// Priority 3: Mixed state (some running, some exited = degraded)
if ($hasRunning && $hasExited) {
return 'degraded:unhealthy';
}
// Priority 4: Running containers (check health status)
if ($hasRunning) {
if ($hasUnhealthy) {
return 'running:unhealthy';
} elseif ($hasUnknown) {
return 'running:unknown';
} else {
return 'running:healthy';
}
}
// Priority 5: Dead or removing containers
if ($hasDead) {
return 'degraded:unhealthy';
}
// Priority 6: Paused containers
if ($hasPaused) {
return 'paused:unknown';
}
// Priority 7: Starting/created containers
if ($hasStarting) {
return 'starting:unknown';
}
// Priority 8: All containers exited (no restart count = truly stopped)
return 'exited:unhealthy';
}
}
app/Traits/CalculatesExcludedStatus.php
+166
View File
@@ -0,0 +1,166 @@
<?php
namespace App\Traits;
use App\Services\ContainerStatusAggregator;
use Illuminate\Support\Collection;
use Illuminate\Support\Facades\Log;
use Symfony\Component\Yaml\Exception\ParseException;
trait CalculatesExcludedStatus
{
/**
* Calculate status for containers when all containers are excluded from health checks.
*
* This method processes excluded containers and returns a status with :excluded suffix
* to indicate that monitoring is disabled but still show the actual container state.
*
* @param Collection $containers Collection of container objects from Docker inspect
* @param Collection $excludedContainers Collection of container names that are excluded
* @return string Status string with :excluded suffix (e.g., 'running:unhealthy:excluded')
*/
protected function calculateExcludedStatus(Collection $containers, Collection $excludedContainers): string
{
// Filter to only excluded containers
$excludedOnly = $containers->filter(function ($container) use ($excludedContainers) {
$labels = data_get($container, 'Config.Labels', []);
$serviceName = data_get($labels, 'com.docker.compose.service');
return $serviceName && $excludedContainers->contains($serviceName);
});
// Use ContainerStatusAggregator service for state machine logic
$aggregator = new ContainerStatusAggregator;
$status = $aggregator->aggregateFromContainers($excludedOnly);
// Append :excluded suffix
return $this->appendExcludedSuffix($status);
}
/**
* Calculate status for containers when all containers are excluded (simplified version).
*
* This version works with status strings (e.g., "running:healthy") instead of full
* container objects, suitable for Sentinel updates that don't have full container data.
*
* @param Collection $containerStatuses Collection of status strings keyed by container name
* @return string Status string with :excluded suffix
*/
protected function calculateExcludedStatusFromStrings(Collection $containerStatuses): string
{
// Use ContainerStatusAggregator service for state machine logic
$aggregator = new ContainerStatusAggregator;
$status = $aggregator->aggregateFromStrings($containerStatuses);
// Append :excluded suffix
$finalStatus = $this->appendExcludedSuffix($status);
return $finalStatus;
}
/**
* Append :excluded suffix to a status string.
*
* Converts status formats like:
* - "running:healthy" → "running:healthy:excluded"
* - "degraded:unhealthy" → "degraded:excluded" (simplified)
* - "paused:unknown" → "paused:excluded" (simplified)
*
* @param string $status The base status string
* @return string Status with :excluded suffix
*/
private function appendExcludedSuffix(string $status): string
{
// For degraded states, simplify to just "degraded:excluded"
if (str($status)->startsWith('degraded')) {
return 'degraded:excluded';
}
// For paused/starting/exited states, simplify to just "state:excluded"
if (str($status)->startsWith('paused')) {
return 'paused:excluded';
}
if (str($status)->startsWith('starting')) {
return 'starting:excluded';
}
if (str($status)->startsWith('exited')) {
return 'exited:excluded';
}
// For running states, keep the health status: "running:healthy:excluded"
return "$status:excluded";
}
/**
* Get excluded containers from docker-compose YAML.
*
* Containers are excluded if:
* - They have exclude_from_hc: true label
* - They have restart: no policy
*
* @param string|null $dockerComposeRaw The raw docker-compose YAML content
* @return Collection Collection of excluded container names
*/
protected function getExcludedContainersFromDockerCompose(?string $dockerComposeRaw): Collection
{
$excludedContainers = collect();
if (! $dockerComposeRaw) {
return $excludedContainers;
}
try {
$dockerCompose = \Symfony\Component\Yaml\Yaml::parse($dockerComposeRaw);
// Validate structure
if (! is_array($dockerCompose)) {
Log::warning('Docker Compose YAML did not parse to array', [
'yaml_length' => strlen($dockerComposeRaw),
'parsed_type' => gettype($dockerCompose),
]);
return $excludedContainers;
}
$services = data_get($dockerCompose, 'services', []);
if (! is_array($services)) {
Log::warning('Docker Compose services is not an array', [
'services_type' => gettype($services),
]);
return $excludedContainers;
}
foreach ($services as $serviceName => $serviceConfig) {
$excludeFromHc = data_get($serviceConfig, 'exclude_from_hc', false);
$restartPolicy = data_get($serviceConfig, 'restart', 'always');
if ($excludeFromHc || $restartPolicy === 'no') {
$excludedContainers->push($serviceName);
}
}
} catch (ParseException $e) {
// Specific YAML parsing errors
Log::warning('Failed to parse Docker Compose YAML for health check exclusions', [
'error' => $e->getMessage(),
'line' => $e->getParsedLine(),
'snippet' => $e->getSnippet(),
]);
return $excludedContainers;
} catch (\Exception $e) {
// Unexpected errors
Log::error('Unexpected error parsing Docker Compose YAML', [
'error' => $e->getMessage(),
'trace' => $e->getTraceAsString(),
]);
return $excludedContainers;
}
return $excludedContainers;
}
}