tiennm99/coolify
1
0 Fork 0
mirror of https://github.com/tiennm99/coolify.git synced 2026-10-16 16:15:36 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(tests): add comprehensive tests for ContainerStatusAggregator and serverStatus accessor

Browse Source 
- Introduced tests for ContainerStatusAggregator to validate status aggregation logic across various container states.
- Implemented tests to ensure serverStatus accessor correctly checks server infrastructure health without being affected by container status.
- Updated ExcludeFromHealthCheckTest to verify excluded status handling in various components.
- Removed obsolete PushServerUpdateJobStatusAggregationTest as its functionality is covered elsewhere.
- Updated version number for sentinel to 0.0.17 in versions.json.
This commit is contained in:
Andras Bacsai committed 2025-11-20 17:31:07 +01:00
1 parent 70fb4c6869
commit ae6eef3cdb
23 files changed
+1589 -911
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

app/Actions/Docker/GetContainersStatus.php
+26 -169
View File
@@ -8,6 +8,8 @@ use App\Events\ServiceChecked;
use App\Models\ApplicationPreview;
use App\Models\Server;
use App\Models\ServiceDatabase;
use App\Services\ContainerStatusAggregator;
use App\Traits\CalculatesExcludedStatus;
use Illuminate\Support\Arr;
use Illuminate\Support\Collection;
use Illuminate\Support\Facades\DB;
@@ -16,6 +18,7 @@ use Lorisleiva\Actions\Concerns\AsAction;
class GetContainersStatus
{
use AsAction;
use CalculatesExcludedStatus;
public string $jobQueue = 'high';
@@ -103,10 +106,10 @@ class GetContainersStatus
$containerHealth = data_get($container, 'State.Health.Status');
if ($containerStatus === 'restarting') {
$healthSuffix = $containerHealth ?? 'unknown';
$containerStatus = "restarting ($healthSuffix)";
$containerStatus = "restarting:$healthSuffix";
} else {
$healthSuffix = $containerHealth ?? 'unknown';
$containerStatus = "$containerStatus ($healthSuffix)";
$containerStatus = "$containerStatus:$healthSuffix";
}
$labels = Arr::undot(format_docker_labels_to_json($labels));
$applicationId = data_get($labels, 'coolify.applicationId');
@@ -443,106 +446,22 @@ class GetContainersStatus
{
// Parse docker compose to check for excluded containers
$dockerComposeRaw = data_get($application, 'docker_compose_raw');
$excludedContainers = collect();
if ($dockerComposeRaw) {
try {
$dockerCompose = \Symfony\Component\Yaml\Yaml::parse($dockerComposeRaw);
$services = data_get($dockerCompose, 'services', []);
foreach ($services as $serviceName => $serviceConfig) {
// Check if container should be excluded
$excludeFromHc = data_get($serviceConfig, 'exclude_from_hc', false);
$restartPolicy = data_get($serviceConfig, 'restart', 'always');
if ($excludeFromHc || $restartPolicy === 'no') {
$excludedContainers->push($serviceName);
}
}
} catch (\Exception $e) {
// If we can't parse, treat all containers as included
}
}
$excludedContainers = $this->getExcludedContainersFromDockerCompose($dockerComposeRaw);
// Filter out excluded containers
$relevantStatuses = $containerStatuses->filter(function ($status, $containerName) use ($excludedContainers) {
return ! $excludedContainers->contains($containerName);
});
// If all containers are excluded, don't update status
// If all containers are excluded, calculate status from excluded containers
if ($relevantStatuses->isEmpty()) {
return null;
return $this->calculateExcludedStatusFromStrings($containerStatuses);
}
$hasRunning = false;
$hasRestarting = false;
$hasUnhealthy = false;
$hasUnknown = false;
$hasExited = false;
$hasStarting = false;
$hasPaused = false;
$hasDead = false;
// Use ContainerStatusAggregator service for state machine logic
$aggregator = new ContainerStatusAggregator;
foreach ($relevantStatuses as $status) {
if (str($status)->contains('restarting')) {
$hasRestarting = true;
} elseif (str($status)->contains('running')) {
$hasRunning = true;
if (str($status)->contains('unhealthy')) {
$hasUnhealthy = true;
}
if (str($status)->contains('unknown')) {
$hasUnknown = true;
}
} elseif (str($status)->contains('exited')) {
$hasExited = true;
$hasUnhealthy = true;
} elseif (str($status)->contains('created') || str($status)->contains('starting')) {
$hasStarting = true;
} elseif (str($status)->contains('paused')) {
$hasPaused = true;
} elseif (str($status)->contains('dead') || str($status)->contains('removing')) {
$hasDead = true;
}
}
if ($hasRestarting) {
return 'degraded (unhealthy)';
}
// If container is exited but has restart count > 0, it's in a crash loop
if ($hasExited && $maxRestartCount > 0) {
return 'degraded (unhealthy)';
}
if ($hasRunning && $hasExited) {
return 'degraded (unhealthy)';
}
if ($hasRunning) {
if ($hasUnhealthy) {
return 'running (unhealthy)';
} elseif ($hasUnknown) {
return 'running (unknown)';
} else {
return 'running (healthy)';
}
}
if ($hasDead) {
return 'degraded (unhealthy)';
}
if ($hasPaused) {
return 'paused (unknown)';
}
if ($hasStarting) {
return 'starting (unknown)';
}
// All containers are exited with no restart count - truly stopped
return 'exited (unhealthy)';
return $aggregator->aggregateFromStrings($relevantStatuses, $maxRestartCount);
}
private function aggregateServiceContainerStatuses($services)
@@ -574,93 +493,31 @@ class GetContainersStatus
// Parse docker compose from service to check for excluded containers
$dockerComposeRaw = data_get($service, 'docker_compose_raw');
$excludedContainers = collect();
if ($dockerComposeRaw) {
try {
$dockerCompose = \Symfony\Component\Yaml\Yaml::parse($dockerComposeRaw);
$servicesInCompose = data_get($dockerCompose, 'services', []);
foreach ($servicesInCompose as $serviceName => $serviceConfig) {
// Check if container should be excluded
$excludeFromHc = data_get($serviceConfig, 'exclude_from_hc', false);
$restartPolicy = data_get($serviceConfig, 'restart', 'always');
if ($excludeFromHc || $restartPolicy === 'no') {
$excludedContainers->push($serviceName);
}
}
} catch (\Exception $e) {
// If we can't parse, treat all containers as included
}
}
$excludedContainers = $this->getExcludedContainersFromDockerCompose($dockerComposeRaw);
// Filter out excluded containers
$relevantStatuses = $containerStatuses->filter(function ($status, $containerName) use ($excludedContainers) {
return ! $excludedContainers->contains($containerName);
});
// If all containers are excluded, don't update status
// If all containers are excluded, calculate status from excluded containers
if ($relevantStatuses->isEmpty()) {
$aggregatedStatus = $this->calculateExcludedStatusFromStrings($containerStatuses);
if ($aggregatedStatus) {
$statusFromDb = $subResource->status;
if ($statusFromDb !== $aggregatedStatus) {
$subResource->update(['status' => $aggregatedStatus]);
} else {
$subResource->update(['last_online_at' => now()]);
}
}
continue;
}
// Aggregate status using same logic as applications
$hasRunning = false;
$hasRestarting = false;
$hasUnhealthy = false;
$hasUnknown = false;
$hasExited = false;
$hasStarting = false;
$hasPaused = false;
$hasDead = false;
foreach ($relevantStatuses as $status) {
if (str($status)->contains('restarting')) {
$hasRestarting = true;
} elseif (str($status)->contains('running')) {
$hasRunning = true;
if (str($status)->contains('unhealthy')) {
$hasUnhealthy = true;
}
if (str($status)->contains('unknown')) {
$hasUnknown = true;
}
} elseif (str($status)->contains('exited')) {
$hasExited = true;
$hasUnhealthy = true;
} elseif (str($status)->contains('created') || str($status)->contains('starting')) {
$hasStarting = true;
} elseif (str($status)->contains('paused')) {
$hasPaused = true;
} elseif (str($status)->contains('dead') || str($status)->contains('removing')) {
$hasDead = true;
}
}
$aggregatedStatus = null;
if ($hasRestarting) {
$aggregatedStatus = 'degraded (unhealthy)';
} elseif ($hasRunning && $hasExited) {
$aggregatedStatus = 'degraded (unhealthy)';
} elseif ($hasRunning) {
if ($hasUnhealthy) {
$aggregatedStatus = 'running (unhealthy)';
} elseif ($hasUnknown) {
$aggregatedStatus = 'running (unknown)';
} else {
$aggregatedStatus = 'running (healthy)';
}
} elseif ($hasDead) {
$aggregatedStatus = 'degraded (unhealthy)';
} elseif ($hasPaused) {
$aggregatedStatus = 'paused (unknown)';
} elseif ($hasStarting) {
$aggregatedStatus = 'starting (unknown)';
} else {
// All containers are exited
$aggregatedStatus = 'exited (unhealthy)';
}
// Use ContainerStatusAggregator service for state machine logic
$aggregator = new ContainerStatusAggregator;
$aggregatedStatus = $aggregator->aggregateFromStrings($relevantStatuses);
// Update service sub-resource status with aggregated result
if ($aggregatedStatus) {
app/Actions/Shared/ComplexStatusCheck.php
+13 -164
View File
@@ -3,11 +3,14 @@
namespace App\Actions\Shared;
use App\Models\Application;
use App\Services\ContainerStatusAggregator;
use App\Traits\CalculatesExcludedStatus;
use Lorisleiva\Actions\Concerns\AsAction;
class ComplexStatusCheck
{
use AsAction;
use CalculatesExcludedStatus;
public function handle(Application $application)
{
@@ -61,179 +64,25 @@ class ComplexStatusCheck
private function aggregateContainerStatuses($application, $containers)
{
$dockerComposeRaw = data_get($application, 'docker_compose_raw');
$excludedContainers = collect();
$excludedContainers = $this->getExcludedContainersFromDockerCompose($dockerComposeRaw);
if ($dockerComposeRaw) {
try {
$dockerCompose = \Symfony\Component\Yaml\Yaml::parse($dockerComposeRaw);
$services = data_get($dockerCompose, 'services', []);
foreach ($services as $serviceName => $serviceConfig) {
$excludeFromHc = data_get($serviceConfig, 'exclude_from_hc', false);
$restartPolicy = data_get($serviceConfig, 'restart', 'always');
if ($excludeFromHc || $restartPolicy === 'no') {
$excludedContainers->push($serviceName);
}
}
} catch (\Exception $e) {
// If we can't parse, treat all containers as included
}
}
$hasRunning = false;
$hasRestarting = false;
$hasUnhealthy = false;
$hasUnknown = false;
$hasExited = false;
$hasStarting = false;
$hasPaused = false;
$hasDead = false;
$relevantContainerCount = 0;
foreach ($containers as $container) {
// Filter non-excluded containers
$relevantContainers = collect($containers)->filter(function ($container) use ($excludedContainers) {
$labels = data_get($container, 'Config.Labels', []);
$serviceName = data_get($labels, 'com.docker.compose.service');
if ($serviceName && $excludedContainers->contains($serviceName)) {
continue;
}
$relevantContainerCount++;
$containerStatus = data_get($container, 'State.Status');
$containerHealth = data_get($container, 'State.Health.Status');
if ($containerStatus === 'restarting') {
$hasRestarting = true;
$hasUnhealthy = true;
} elseif ($containerStatus === 'running') {
$hasRunning = true;
if ($containerHealth === 'unhealthy') {
$hasUnhealthy = true;
} elseif ($containerHealth === null) {
$hasUnknown = true;
}
} elseif ($containerStatus === 'exited') {
$hasExited = true;
$hasUnhealthy = true;
} elseif ($containerStatus === 'created' || $containerStatus === 'starting') {
$hasStarting = true;
} elseif ($containerStatus === 'paused') {
$hasPaused = true;
} elseif ($containerStatus === 'dead' || $containerStatus === 'removing') {
$hasDead = true;
}
}
return ! ($serviceName && $excludedContainers->contains($serviceName));
});
// If all containers are excluded, calculate status from excluded containers
// but mark it with :excluded to indicate monitoring is disabled
if ($relevantContainerCount === 0) {
$excludedHasRunning = false;
$excludedHasRestarting = false;
$excludedHasUnhealthy = false;
$excludedHasUnknown = false;
$excludedHasExited = false;
$excludedHasStarting = false;
$excludedHasPaused = false;
$excludedHasDead = false;
foreach ($containers as $container) {
$labels = data_get($container, 'Config.Labels', []);
$serviceName = data_get($labels, 'com.docker.compose.service');
// Only process excluded containers
if (! $serviceName || ! $excludedContainers->contains($serviceName)) {
continue;
}
$containerStatus = data_get($container, 'State.Status');
$containerHealth = data_get($container, 'State.Health.Status');
if ($containerStatus === 'restarting') {
$excludedHasRestarting = true;
$excludedHasUnhealthy = true;
} elseif ($containerStatus === 'running') {
$excludedHasRunning = true;
if ($containerHealth === 'unhealthy') {
$excludedHasUnhealthy = true;
} elseif ($containerHealth === null) {
$excludedHasUnknown = true;
}
} elseif ($containerStatus === 'exited') {
$excludedHasExited = true;
$excludedHasUnhealthy = true;
} elseif ($containerStatus === 'created' || $containerStatus === 'starting') {
$excludedHasStarting = true;
} elseif ($containerStatus === 'paused') {
$excludedHasPaused = true;
} elseif ($containerStatus === 'dead' || $containerStatus === 'removing') {
$excludedHasDead = true;
}
}
if ($excludedHasRestarting) {
return 'degraded:excluded';
}
if ($excludedHasRunning && $excludedHasExited) {
return 'degraded:excluded';
}
if ($excludedHasRunning) {
if ($excludedHasUnhealthy) {
return 'running:unhealthy:excluded';
} elseif ($excludedHasUnknown) {
return 'running:unknown:excluded';
} else {
return 'running:healthy:excluded';
}
}
if ($excludedHasDead) {
return 'degraded:excluded';
}
if ($excludedHasPaused) {
return 'paused:excluded';
}
if ($excludedHasStarting) {
return 'starting:excluded';
}
return 'exited:excluded';
if ($relevantContainers->isEmpty()) {
return $this->calculateExcludedStatus($containers, $excludedContainers);
}
if ($hasRestarting) {
return 'degraded:unhealthy';
}
// Use ContainerStatusAggregator service for state machine logic
$aggregator = new ContainerStatusAggregator;
if ($hasRunning && $hasExited) {
return 'degraded:unhealthy';
}
if ($hasRunning) {
if ($hasUnhealthy) {
return 'running:unhealthy';
} elseif ($hasUnknown) {
return 'running:unknown';
} else {
return 'running:healthy';
}
}
if ($hasDead) {
return 'degraded:unhealthy';
}
if ($hasPaused) {
return 'paused:unknown';
}
if ($hasStarting) {
return 'starting:unknown';
}
return 'exited:unhealthy';
return $aggregator->aggregateFromContainers($relevantContainers);
}
}
app/Jobs/PushServerUpdateJob.php
+90 -172
View File
@@ -13,6 +13,8 @@ use App\Models\Server;
use App\Models\ServiceApplication;
use App\Models\ServiceDatabase;
use App\Notifications\Container\ContainerRestarted;
use App\Services\ContainerStatusAggregator;
use App\Traits\CalculatesExcludedStatus;
use Illuminate\Bus\Queueable;
use Illuminate\Contracts\Queue\ShouldBeEncrypted;
use Illuminate\Contracts\Queue\ShouldQueue;
@@ -25,6 +27,7 @@ use Laravel\Horizon\Contracts\Silenced;
class PushServerUpdateJob implements ShouldBeEncrypted, ShouldQueue, Silenced
{
use CalculatesExcludedStatus;
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
public $tries = 1;
@@ -145,7 +148,7 @@ class PushServerUpdateJob implements ShouldBeEncrypted, ShouldQueue, Silenced
$containerStatus = data_get($container, 'state', 'exited');
$rawHealthStatus = data_get($container, 'health_status');
$containerHealth = $rawHealthStatus ?? 'unknown';
$containerStatus = "$containerStatus ($containerHealth)";
$containerStatus = "$containerStatus:$containerHealth";
$labels = collect(data_get($container, 'labels'));
$coolify_managed = $labels->has('coolify.managed');
@@ -153,81 +156,75 @@ class PushServerUpdateJob implements ShouldBeEncrypted, ShouldQueue, Silenced
continue;
}
if ($coolify_managed) {
$name = data_get($container, 'name');
if ($name === 'coolify-log-drain' && $this->isRunning($containerStatus)) {
$this->foundLogDrainContainer = true;
}
if ($labels->has('coolify.applicationId')) {
$applicationId = $labels->get('coolify.applicationId');
$pullRequestId = $labels->get('coolify.pullRequestId', '0');
try {
if ($pullRequestId === '0') {
if ($this->allApplicationIds->contains($applicationId) && $this->isRunning($containerStatus)) {
$this->foundApplicationIds->push($applicationId);
}
// Store container status for aggregation
if (! $this->applicationContainerStatuses->has($applicationId)) {
$this->applicationContainerStatuses->put($applicationId, collect());
}
$containerName = $labels->get('com.docker.compose.service');
if ($containerName) {
$this->applicationContainerStatuses->get($applicationId)->put($containerName, $containerStatus);
}
} else {
$previewKey = $applicationId.':'.$pullRequestId;
if ($this->allApplicationPreviewsIds->contains($previewKey) && $this->isRunning($containerStatus)) {
$this->foundApplicationPreviewsIds->push($previewKey);
}
$this->updateApplicationPreviewStatus($applicationId, $pullRequestId, $containerStatus);
}
} catch (\Exception $e) {
}
} elseif ($labels->has('coolify.serviceId')) {
$serviceId = $labels->get('coolify.serviceId');
$subType = $labels->get('coolify.service.subType');
$subId = $labels->get('coolify.service.subId');
if ($subType === 'application') {
if ($this->isRunning($containerStatus)) {
$this->foundServiceApplicationIds->push($subId);
$name = data_get($container, 'name');
if ($name === 'coolify-log-drain' && $this->isRunning($containerStatus)) {
$this->foundLogDrainContainer = true;
}
if ($labels->has('coolify.applicationId')) {
$applicationId = $labels->get('coolify.applicationId');
$pullRequestId = $labels->get('coolify.pullRequestId', '0');
try {
if ($pullRequestId === '0') {
if ($this->allApplicationIds->contains($applicationId)) {
$this->foundApplicationIds->push($applicationId);
}
// Store container status for aggregation
$key = $serviceId.':'.$subType.':'.$subId;
if (! $this->serviceContainerStatuses->has($key)) {
$this->serviceContainerStatuses->put($key, collect());
if (! $this->applicationContainerStatuses->has($applicationId)) {
$this->applicationContainerStatuses->put($applicationId, collect());
}
$containerName = $labels->get('com.docker.compose.service');
if ($containerName) {
$this->serviceContainerStatuses->get($key)->put($containerName, $containerStatus);
$this->applicationContainerStatuses->get($applicationId)->put($containerName, $containerStatus);
}
} elseif ($subType === 'database') {
if ($this->isRunning($containerStatus)) {
$this->foundServiceDatabaseIds->push($subId);
}
// Store container status for aggregation
$key = $serviceId.':'.$subType.':'.$subId;
if (! $this->serviceContainerStatuses->has($key)) {
$this->serviceContainerStatuses->put($key, collect());
}
$containerName = $labels->get('com.docker.compose.service');
if ($containerName) {
$this->serviceContainerStatuses->get($key)->put($containerName, $containerStatus);
}
}
} else {
$uuid = $labels->get('com.docker.compose.service');
$type = $labels->get('coolify.type');
if ($name === 'coolify-proxy' && $this->isRunning($containerStatus)) {
$this->foundProxy = true;
} elseif ($type === 'service' && $this->isRunning($containerStatus)) {
} else {
if ($this->allDatabaseUuids->contains($uuid) && $this->isRunning($containerStatus)) {
$this->foundDatabaseUuids->push($uuid);
if ($this->allTcpProxyUuids->contains($uuid) && $this->isRunning($containerStatus)) {
$this->updateDatabaseStatus($uuid, $containerStatus, tcpProxy: true);
} else {
$this->updateDatabaseStatus($uuid, $containerStatus, tcpProxy: false);
}
$previewKey = $applicationId.':'.$pullRequestId;
if ($this->allApplicationPreviewsIds->contains($previewKey)) {
$this->foundApplicationPreviewsIds->push($previewKey);
}
$this->updateApplicationPreviewStatus($applicationId, $pullRequestId, $containerStatus);
}
} catch (\Exception $e) {
}
} elseif ($labels->has('coolify.serviceId')) {
$serviceId = $labels->get('coolify.serviceId');
$subType = $labels->get('coolify.service.subType');
$subId = $labels->get('coolify.service.subId');
if ($subType === 'application') {
$this->foundServiceApplicationIds->push($subId);
// Store container status for aggregation
$key = $serviceId.':'.$subType.':'.$subId;
if (! $this->serviceContainerStatuses->has($key)) {
$this->serviceContainerStatuses->put($key, collect());
}
$containerName = $labels->get('com.docker.compose.service');
if ($containerName) {
$this->serviceContainerStatuses->get($key)->put($containerName, $containerStatus);
}
} elseif ($subType === 'database') {
$this->foundServiceDatabaseIds->push($subId);
// Store container status for aggregation
$key = $serviceId.':'.$subType.':'.$subId;
if (! $this->serviceContainerStatuses->has($key)) {
$this->serviceContainerStatuses->put($key, collect());
}
$containerName = $labels->get('com.docker.compose.service');
if ($containerName) {
$this->serviceContainerStatuses->get($key)->put($containerName, $containerStatus);
}
}
} else {
$uuid = $labels->get('com.docker.compose.service');
$type = $labels->get('coolify.type');
if ($name === 'coolify-proxy' && $this->isRunning($containerStatus)) {
$this->foundProxy = true;
} elseif ($type === 'service' && $this->isRunning($containerStatus)) {
} else {
if ($this->allDatabaseUuids->contains($uuid) && $this->isRunning($containerStatus)) {
$this->foundDatabaseUuids->push($uuid);
if ($this->allTcpProxyUuids->contains($uuid) && $this->isRunning($containerStatus)) {
$this->updateDatabaseStatus($uuid, $containerStatus, tcpProxy: true);
} else {
$this->updateDatabaseStatus($uuid, $containerStatus, tcpProxy: false);
}
}
}
@@ -266,71 +263,31 @@ class PushServerUpdateJob implements ShouldBeEncrypted, ShouldQueue, Silenced
// Parse docker compose to check for excluded containers
$dockerComposeRaw = data_get($application, 'docker_compose_raw');
$excludedContainers = collect();
if ($dockerComposeRaw) {
try {
$dockerCompose = \Symfony\Component\Yaml\Yaml::parse($dockerComposeRaw);
$services = data_get($dockerCompose, 'services', []);
foreach ($services as $serviceName => $serviceConfig) {
// Check if container should be excluded
$excludeFromHc = data_get($serviceConfig, 'exclude_from_hc', false);
$restartPolicy = data_get($serviceConfig, 'restart', 'always');
if ($excludeFromHc || $restartPolicy === 'no') {
$excludedContainers->push($serviceName);
}
}
} catch (\Exception $e) {
// If we can't parse, treat all containers as included
}
}
$excludedContainers = $this->getExcludedContainersFromDockerCompose($dockerComposeRaw);
// Filter out excluded containers
$relevantStatuses = $containerStatuses->filter(function ($status, $containerName) use ($excludedContainers) {
return ! $excludedContainers->contains($containerName);
});
// If all containers are excluded, don't update status
// If all containers are excluded, calculate status from excluded containers
if ($relevantStatuses->isEmpty()) {
$aggregatedStatus = $this->calculateExcludedStatusFromStrings($containerStatuses);
if ($aggregatedStatus && $application->status !== $aggregatedStatus) {
$application->status = $aggregatedStatus;
$application->save();
}
continue;
}
// Aggregate status: if any container is running, app is running
$hasRunning = false;
$hasUnhealthy = false;
$hasUnknown = false;
foreach ($relevantStatuses as $status) {
if (str($status)->contains('running')) {
$hasRunning = true;
if (str($status)->contains('unhealthy')) {
$hasUnhealthy = true;
}
if (str($status)->contains('unknown')) {
$hasUnknown = true;
}
}
}
$aggregatedStatus = null;
if ($hasRunning) {
if ($hasUnhealthy) {
$aggregatedStatus = 'running (unhealthy)';
} elseif ($hasUnknown) {
$aggregatedStatus = 'running (unknown)';
} else {
$aggregatedStatus = 'running (healthy)';
}
} else {
// All containers are exited
$aggregatedStatus = 'exited (unhealthy)';
}
// Use ContainerStatusAggregator service for state machine logic
$aggregator = new ContainerStatusAggregator;
$aggregatedStatus = $aggregator->aggregateFromStrings($relevantStatuses, 0);
// Update application status with aggregated result
if ($aggregatedStatus && $application->status !== $aggregatedStatus) {
$application->status = $aggregatedStatus;
$application->save();
}
@@ -366,67 +323,28 @@ class PushServerUpdateJob implements ShouldBeEncrypted, ShouldQueue, Silenced
// Parse docker compose from service to check for excluded containers
$dockerComposeRaw = data_get($service, 'docker_compose_raw');
$excludedContainers = collect();
if ($dockerComposeRaw) {
try {
$dockerCompose = \Symfony\Component\Yaml\Yaml::parse($dockerComposeRaw);
$services = data_get($dockerCompose, 'services', []);
foreach ($services as $serviceName => $serviceConfig) {
// Check if container should be excluded
$excludeFromHc = data_get($serviceConfig, 'exclude_from_hc', false);
$restartPolicy = data_get($serviceConfig, 'restart', 'always');
if ($excludeFromHc || $restartPolicy === 'no') {
$excludedContainers->push($serviceName);
}
}
} catch (\Exception $e) {
// If we can't parse, treat all containers as included
}
}
$excludedContainers = $this->getExcludedContainersFromDockerCompose($dockerComposeRaw);
// Filter out excluded containers
$relevantStatuses = $containerStatuses->filter(function ($status, $containerName) use ($excludedContainers) {
return ! $excludedContainers->contains($containerName);
});
// If all containers are excluded, don't update status
// If all containers are excluded, calculate status from excluded containers
if ($relevantStatuses->isEmpty()) {
$aggregatedStatus = $this->calculateExcludedStatusFromStrings($containerStatuses);
if ($aggregatedStatus && $subResource->status !== $aggregatedStatus) {
$subResource->status = $aggregatedStatus;
$subResource->save();
}
continue;
}
// Aggregate status: if any container is running, service is running
$hasRunning = false;
$hasUnhealthy = false;
$hasUnknown = false;
foreach ($relevantStatuses as $status) {
if (str($status)->contains('running')) {
$hasRunning = true;
if (str($status)->contains('unhealthy')) {
$hasUnhealthy = true;
}
if (str($status)->contains('unknown')) {
$hasUnknown = true;
}
}
}
$aggregatedStatus = null;
if ($hasRunning) {
if ($hasUnhealthy) {
$aggregatedStatus = 'running (unhealthy)';
} elseif ($hasUnknown) {
$aggregatedStatus = 'running (unknown)';
} else {
$aggregatedStatus = 'running (healthy)';
}
} else {
// All containers are exited
$aggregatedStatus = 'exited (unhealthy)';
}
// Use ContainerStatusAggregator service for state machine logic
// NOTE: Sentinel does NOT provide restart count data, so maxRestartCount is always 0
$aggregator = new ContainerStatusAggregator;
$aggregatedStatus = $aggregator->aggregateFromStrings($relevantStatuses, 0);
// Update service sub-resource status with aggregated result
if ($aggregatedStatus && $subResource->status !== $aggregatedStatus) {
app/Models/Application.php
+12 -25
View File
@@ -12,7 +12,6 @@ use Illuminate\Database\Eloquent\Factories\HasFactory;
use Illuminate\Database\Eloquent\Relations\HasMany;
use Illuminate\Database\Eloquent\SoftDeletes;
use Illuminate\Support\Collection;
use Illuminate\Support\Facades\Log;
use Illuminate\Support\Facades\Validator;
use Illuminate\Support\Str;
use OpenApi\Attributes as OA;
@@ -670,21 +669,23 @@ class Application extends BaseModel
{
return Attribute::make(
get: function () {
if (! $this->relationLoaded('additional_servers') || $this->additional_servers->count() === 0) {
return $this->destination?->server?->isFunctional() ?? false;
// Check main server infrastructure health
$main_server_functional = $this->destination?->server?->isFunctional() ?? false;
if (! $main_server_functional) {
return false;
}
$additional_servers_status = $this->additional_servers->pluck('pivot.status');
$main_server_status = $this->destination?->server?->isFunctional() ?? false;
foreach ($additional_servers_status as $status) {
$server_status = str($status)->before(':')->value();
if ($server_status !== 'running') {
return false;
// Check additional servers infrastructure health (not container status!)
if ($this->relationLoaded('additional_servers') && $this->additional_servers->count() > 0) {
foreach ($this->additional_servers as $server) {
if (! $server->isFunctional()) {
return false; // Real server infrastructure problem
}
}
}
return $main_server_status;
return true;
}
);
}
@@ -703,13 +704,6 @@ class Application extends BaseModel
} else {
$status = $value;
$health = 'unhealthy';
Log::debug('[STATUS-DEBUG] Status set without health - defaulting to unhealthy', [
'source' => 'Application model accessor',
'app_id' => $this->id,
'app_name' => $this->name,
'raw_value' => $value,
'result' => "$status:$health",
]);
}
return "$status:$health";
@@ -723,13 +717,6 @@ class Application extends BaseModel
} else {
$status = $value;
$health = 'unhealthy';
Log::debug('[STATUS-DEBUG] Status set without health (multi-server) - defaulting to unhealthy', [
'source' => 'Application model accessor',
'app_id' => $this->id,
'app_name' => $this->name,
'raw_value' => $value,
'result' => "$status:$health",
]);
}
return "$status:$health";
app/Services/ContainerStatusAggregator.php
+222
View File
@@ -0,0 +1,222 @@
<?php
namespace App\Services;
use Illuminate\Support\Collection;
/**
* Container Status Aggregator Service
*
* Centralized service for aggregating container statuses into a single status string.
* Uses a priority-based state machine to determine the overall status from multiple containers.
*
* Output Format: Colon-separated (e.g., "running:healthy", "degraded:unhealthy")
* This format is used throughout the backend for consistency and machine-readability.
* UI components transform this to human-readable format (e.g., "Running (Healthy)").
*
* State Priority (highest to lowest):
* 1. Restarting → degraded:unhealthy
* 2. Crash Loop (exited with restarts) → degraded:unhealthy
* 3. Mixed (running + exited) → degraded:unhealthy
* 4. Running → running:healthy/unhealthy/unknown
* 5. Dead/Removing → degraded:unhealthy
* 6. Paused → paused:unknown
* 7. Starting/Created → starting:unknown
* 8. Exited → exited:unhealthy
*/
class ContainerStatusAggregator
{
/**
* Aggregate container statuses from status strings into a single status.
*
* @param Collection $containerStatuses Collection of status strings (e.g., "running (healthy)", "running:healthy")
* @param int $maxRestartCount Maximum restart count across containers (for crash loop detection)
* @return string Aggregated status in colon format (e.g., "running:healthy")
*/
public function aggregateFromStrings(Collection $containerStatuses, int $maxRestartCount = 0): string
{
if ($containerStatuses->isEmpty()) {
return 'exited:unhealthy';
}
// Initialize state flags
$hasRunning = false;
$hasRestarting = false;
$hasUnhealthy = false;
$hasUnknown = false;
$hasExited = false;
$hasStarting = false;
$hasPaused = false;
$hasDead = false;
// Parse each status string and set flags
foreach ($containerStatuses as $status) {
if (str($status)->contains('restarting')) {
$hasRestarting = true;
} elseif (str($status)->contains('running')) {
$hasRunning = true;
if (str($status)->contains('unhealthy')) {
$hasUnhealthy = true;
}
if (str($status)->contains('unknown')) {
$hasUnknown = true;
}
} elseif (str($status)->contains('exited')) {
$hasExited = true;
$hasUnhealthy = true;
} elseif (str($status)->contains('created') || str($status)->contains('starting')) {
$hasStarting = true;
} elseif (str($status)->contains('paused')) {
$hasPaused = true;
} elseif (str($status)->contains('dead') || str($status)->contains('removing')) {
$hasDead = true;
}
}
// Priority-based status resolution
return $this->resolveStatus(
$hasRunning,
$hasRestarting,
$hasUnhealthy,
$hasUnknown,
$hasExited,
$hasStarting,
$hasPaused,
$hasDead,
$maxRestartCount
);
}
/**
* Aggregate container statuses from Docker container objects.
*
* @param Collection $containers Collection of Docker container objects with State property
* @param int $maxRestartCount Maximum restart count across containers (for crash loop detection)
* @return string Aggregated status in colon format (e.g., "running:healthy")
*/
public function aggregateFromContainers(Collection $containers, int $maxRestartCount = 0): string
{
if ($containers->isEmpty()) {
return 'exited:unhealthy';
}
// Initialize state flags
$hasRunning = false;
$hasRestarting = false;
$hasUnhealthy = false;
$hasUnknown = false;
$hasExited = false;
$hasStarting = false;
$hasPaused = false;
$hasDead = false;
// Parse each container object and set flags
foreach ($containers as $container) {
$state = data_get($container, 'State.Status', 'exited');
$health = data_get($container, 'State.Health.Status');
if ($state === 'restarting') {
$hasRestarting = true;
} elseif ($state === 'running') {
$hasRunning = true;
if ($health === 'unhealthy') {
$hasUnhealthy = true;
} elseif (is_null($health) || $health === 'starting') {
$hasUnknown = true;
}
} elseif ($state === 'exited') {
$hasExited = true;
$hasUnhealthy = true;
} elseif ($state === 'created' || $state === 'starting') {
$hasStarting = true;
} elseif ($state === 'paused') {
$hasPaused = true;
} elseif ($state === 'dead' || $state === 'removing') {
$hasDead = true;
}
}
// Priority-based status resolution
return $this->resolveStatus(
$hasRunning,
$hasRestarting,
$hasUnhealthy,
$hasUnknown,
$hasExited,
$hasStarting,
$hasPaused,
$hasDead,
$maxRestartCount
);
}
/**
* Resolve the aggregated status based on state flags (priority-based state machine).
*
* @param bool $hasRunning Has at least one running container
* @param bool $hasRestarting Has at least one restarting container
* @param bool $hasUnhealthy Has at least one unhealthy container
* @param bool $hasUnknown Has at least one container with unknown health
* @param bool $hasExited Has at least one exited container
* @param bool $hasStarting Has at least one starting/created container
* @param bool $hasPaused Has at least one paused container
* @param bool $hasDead Has at least one dead/removing container
* @param int $maxRestartCount Maximum restart count (for crash loop detection)
* @return string Status in colon format (e.g., "running:healthy")
*/
private function resolveStatus(
bool $hasRunning,
bool $hasRestarting,
bool $hasUnhealthy,
bool $hasUnknown,
bool $hasExited,
bool $hasStarting,
bool $hasPaused,
bool $hasDead,
int $maxRestartCount
): string {
// Priority 1: Restarting containers (degraded state)
if ($hasRestarting) {
return 'degraded:unhealthy';
}
// Priority 2: Crash loop detection (exited with restart count > 0)
if ($hasExited && $maxRestartCount > 0) {
return 'degraded:unhealthy';
}
// Priority 3: Mixed state (some running, some exited = degraded)
if ($hasRunning && $hasExited) {
return 'degraded:unhealthy';
}
// Priority 4: Running containers (check health status)
if ($hasRunning) {
if ($hasUnhealthy) {
return 'running:unhealthy';
} elseif ($hasUnknown) {
return 'running:unknown';
} else {
return 'running:healthy';
}
}
// Priority 5: Dead or removing containers
if ($hasDead) {
return 'degraded:unhealthy';
}
// Priority 6: Paused containers
if ($hasPaused) {
return 'paused:unknown';
}
// Priority 7: Starting/created containers
if ($hasStarting) {
return 'starting:unknown';
}
// Priority 8: All containers exited (no restart count = truly stopped)
return 'exited:unhealthy';
}
}
app/Traits/CalculatesExcludedStatus.php
+130
View File
@@ -0,0 +1,130 @@
<?php
namespace App\Traits;
use App\Services\ContainerStatusAggregator;
use Illuminate\Support\Collection;
trait CalculatesExcludedStatus
{
/**
* Calculate status for containers when all containers are excluded from health checks.
*
* This method processes excluded containers and returns a status with :excluded suffix
* to indicate that monitoring is disabled but still show the actual container state.
*
* @param Collection $containers Collection of container objects from Docker inspect
* @param Collection $excludedContainers Collection of container names that are excluded
* @return string Status string with :excluded suffix (e.g., 'running:unhealthy:excluded')
*/
protected function calculateExcludedStatus(Collection $containers, Collection $excludedContainers): string
{
// Filter to only excluded containers
$excludedOnly = $containers->filter(function ($container) use ($excludedContainers) {
$labels = data_get($container, 'Config.Labels', []);
$serviceName = data_get($labels, 'com.docker.compose.service');
return $serviceName && $excludedContainers->contains($serviceName);
});
// Use ContainerStatusAggregator service for state machine logic
$aggregator = new ContainerStatusAggregator;
$status = $aggregator->aggregateFromContainers($excludedOnly);
// Append :excluded suffix
return $this->appendExcludedSuffix($status);
}
/**
* Calculate status for containers when all containers are excluded (simplified version).
*
* This version works with status strings (e.g., "running:healthy") instead of full
* container objects, suitable for Sentinel updates that don't have full container data.
*
* @param Collection $containerStatuses Collection of status strings keyed by container name
* @return string Status string with :excluded suffix
*/
protected function calculateExcludedStatusFromStrings(Collection $containerStatuses): string
{
// Use ContainerStatusAggregator service for state machine logic
$aggregator = new ContainerStatusAggregator;
$status = $aggregator->aggregateFromStrings($containerStatuses);
// Append :excluded suffix
$finalStatus = $this->appendExcludedSuffix($status);
return $finalStatus;
}
/**
* Append :excluded suffix to a status string.
*
* Converts status formats like:
* - "running:healthy" → "running:healthy:excluded"
* - "degraded:unhealthy" → "degraded:excluded" (simplified)
* - "paused:unknown" → "paused:excluded" (simplified)
*
* @param string $status The base status string
* @return string Status with :excluded suffix
*/
private function appendExcludedSuffix(string $status): string
{
// For degraded states, simplify to just "degraded:excluded"
if (str($status)->startsWith('degraded')) {
return 'degraded:excluded';
}
// For paused/starting/exited states, simplify to just "state:excluded"
if (str($status)->startsWith('paused')) {
return 'paused:excluded';
}
if (str($status)->startsWith('starting')) {
return 'starting:excluded';
}
if (str($status)->startsWith('exited')) {
return 'exited:excluded';
}
// For running states, keep the health status: "running:healthy:excluded"
return "$status:excluded";
}
/**
* Get excluded containers from docker-compose YAML.
*
* Containers are excluded if:
* - They have exclude_from_hc: true label
* - They have restart: no policy
*
* @param string|null $dockerComposeRaw The raw docker-compose YAML content
* @return Collection Collection of excluded container names
*/
protected function getExcludedContainersFromDockerCompose(?string $dockerComposeRaw): Collection
{
$excludedContainers = collect();
if (! $dockerComposeRaw) {
return $excludedContainers;
}
try {
$dockerCompose = \Symfony\Component\Yaml\Yaml::parse($dockerComposeRaw);
$services = data_get($dockerCompose, 'services', []);
foreach ($services as $serviceName => $serviceConfig) {
$excludeFromHc = data_get($serviceConfig, 'exclude_from_hc', false);
$restartPolicy = data_get($serviceConfig, 'restart', 'always');
if ($excludeFromHc || $restartPolicy === 'no') {
$excludedContainers->push($serviceName);
}
}
} catch (\Exception $e) {
// If we can't parse, treat all containers as included
}
return $excludedContainers;
}
}