tiennm99/coolify
1
0 Fork 0
mirror of https://github.com/tiennm99/coolify.git synced 2026-10-13 18:15:24 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/next' into fix/form-state

Browse Source
This commit is contained in:
Andras Bacsai committed 2026-05-25 16:08:19 +02:00
2 parents 7a3fcd37d5 a49bc5dd14
commit 27b76a4e97
65 files changed
+4703 -1945
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

app/Actions/Database/StartDatabase.php
+13 -9
View File
@@ -11,12 +11,16 @@ use App\Models\StandaloneMysql;
use App\Models\StandalonePostgresql;
use App\Models\StandaloneRedis;
use Lorisleiva\Actions\Concerns\AsAction;
use Lorisleiva\Actions\Decorators\JobDecorator;
class StartDatabase
{
use AsAction;
public string $jobQueue = 'high';
public function configureJob(JobDecorator $job): void
{
$job->onQueue(deployment_queue());
}
public function handle(StandaloneRedis|StandalonePostgresql|StandaloneMongodb|StandaloneMysql|StandaloneMariadb|StandaloneKeydb|StandaloneDragonfly|StandaloneClickhouse $database)
{
@@ -25,28 +29,28 @@ class StartDatabase
return 'Server is not functional';
}
switch ($database->getMorphClass()) {
case \App\Models\StandalonePostgresql::class:
case StandalonePostgresql::class:
$activity = StartPostgresql::run($database);
break;
case \App\Models\StandaloneRedis::class:
case StandaloneRedis::class:
$activity = StartRedis::run($database);
break;
case \App\Models\StandaloneMongodb::class:
case StandaloneMongodb::class:
$activity = StartMongodb::run($database);
break;
case \App\Models\StandaloneMysql::class:
case StandaloneMysql::class:
$activity = StartMysql::run($database);
break;
case \App\Models\StandaloneMariadb::class:
case StandaloneMariadb::class:
$activity = StartMariadb::run($database);
break;
case \App\Models\StandaloneKeydb::class:
case StandaloneKeydb::class:
$activity = StartKeydb::run($database);
break;
case \App\Models\StandaloneDragonfly::class:
case StandaloneDragonfly::class:
$activity = StartDragonfly::run($database);
break;
case \App\Models\StandaloneClickhouse::class:
case StandaloneClickhouse::class:
$activity = StartClickhouse::run($database);
break;
}
app/Actions/Database/StartDatabaseProxy.php
+8 -3
View File
@@ -11,14 +11,19 @@ use App\Models\StandaloneMongodb;
use App\Models\StandaloneMysql;
use App\Models\StandalonePostgresql;
use App\Models\StandaloneRedis;
use App\Notifications\Container\ContainerRestarted;
use Lorisleiva\Actions\Concerns\AsAction;
use Lorisleiva\Actions\Decorators\JobDecorator;
use Symfony\Component\Yaml\Yaml;
class StartDatabaseProxy
{
use AsAction;
public string $jobQueue = 'high';
public function configureJob(JobDecorator $job): void
{
$job->onQueue(deployment_queue());
}
public function handle(StandaloneRedis|StandalonePostgresql|StandaloneMongodb|StandaloneMysql|StandaloneMariadb|StandaloneKeydb|StandaloneDragonfly|StandaloneClickhouse|ServiceDatabase $database)
{
@@ -29,7 +34,7 @@ class StartDatabaseProxy
$proxyContainerName = "{$database->uuid}-proxy";
$isSSLEnabled = $database->enable_ssl ?? false;
if ($database->getMorphClass() === \App\Models\ServiceDatabase::class) {
if ($database->getMorphClass() === ServiceDatabase::class) {
$databaseType = $database->databaseType();
$network = $database->service->uuid;
$server = data_get($database, 'service.destination.server');
@@ -132,7 +137,7 @@ class StartDatabaseProxy
?? data_get($database, 'service.environment.project.team');
$team?->notify(
new \App\Notifications\Container\ContainerRestarted(
new ContainerRestarted(
"TCP Proxy for {$database->name} database has been disabled due to error: {$e->getMessage()}",
$server,
)
app/Actions/Service/StartService.php
+5 -1
View File
@@ -4,13 +4,17 @@ namespace App\Actions\Service;
use App\Models\Service;
use Lorisleiva\Actions\Concerns\AsAction;
use Lorisleiva\Actions\Decorators\JobDecorator;
use Symfony\Component\Yaml\Yaml;
class StartService
{
use AsAction;
public string $jobQueue = 'high';
public function configureJob(JobDecorator $job): void
{
$job->onQueue(deployment_queue());
}
public function handle(Service $service, bool $pullLatestImages = false, bool $stopBeforeStart = false)
{
app/Console/Kernel.php
+3 -2
View File
@@ -8,6 +8,7 @@ use App\Jobs\CheckHelperImageJob;
use App\Jobs\CheckTraefikVersionJob;
use App\Jobs\CleanupInstanceStuffsJob;
use App\Jobs\CleanupOrphanedPreviewContainersJob;
use App\Jobs\CleanupStaleMultiplexedConnections;
use App\Jobs\PullChangelog;
use App\Jobs\PullTemplatesFromCDN;
use App\Jobs\RegenerateSslCertJob;
@@ -40,7 +41,7 @@ class Kernel extends ConsoleKernel
$this->instanceTimezone = config('app.timezone');
}
// $this->scheduleInstance->job(new CleanupStaleMultiplexedConnections)->hourly();
$this->scheduleInstance->job(new CleanupStaleMultiplexedConnections)->hourly()->onOneServer();
$this->scheduleInstance->command('cleanup:redis --clear-locks')->daily();
$this->scheduleInstance->command('sanctum:prune-expired --hours=1')->hourly()->onOneServer();
$this->scheduleInstance->job(new ApiTokenExpirationWarningJob)->hourly()->onOneServer();
@@ -78,7 +79,7 @@ class Kernel extends ConsoleKernel
// Scheduled Jobs (Backups & Tasks)
$this->scheduleInstance->job(new ScheduledJobManager)->everyMinute()->onOneServer();
$this->scheduleInstance->job(new RegenerateSslCertJob)->twiceDaily();
$this->scheduleInstance->job(new RegenerateSslCertJob)->twiceDaily()->onOneServer();
$this->scheduleInstance->job(new CheckTraefikVersionJob)->weekly()->sundays()->at('00:00')->timezone($this->instanceTimezone)->onOneServer();
app/Helpers/SshMultiplexingHelper.php
+46 -212
View File
@@ -4,7 +4,6 @@ namespace App\Helpers;
use App\Models\PrivateKey;
use App\Models\Server;
use Illuminate\Support\Facades\Cache;
use Illuminate\Support\Facades\Hash;
use Illuminate\Support\Facades\Log;
use Illuminate\Support\Facades\Process;
@@ -12,145 +11,65 @@ use Illuminate\Support\Facades\Storage;
class SshMultiplexingHelper
{
public static function serverSshConfiguration(Server $server)
public static function serverSshConfiguration(Server $server): array
{
$privateKey = PrivateKey::findOrFail($server->private_key_id);
$sshKeyLocation = $privateKey->getKeyLocation();
$muxFilename = '/var/www/html/storage/app/ssh/mux/mux_'.$server->uuid;
return [
'sshKeyLocation' => $sshKeyLocation,
'muxFilename' => $muxFilename,
'sshKeyLocation' => $privateKey->getKeyLocation(),
'muxFilename' => self::muxSocket($server),
];
}
public static function ensureMultiplexedConnection(Server $server): bool
{
if (! self::isMultiplexingEnabled()) {
return false;
}
$sshConfig = self::serverSshConfiguration($server);
$muxSocket = $sshConfig['muxFilename'];
// Check if connection exists
$checkCommand = "ssh -O check -o ControlPath=$muxSocket ";
if (data_get($server, 'settings.is_cloudflare_tunnel')) {
$checkCommand .= '-o ProxyCommand="cloudflared access ssh --hostname %h" ';
}
$checkCommand .= self::escapedUserAtHost($server);
$process = Process::run($checkCommand);
if ($process->exitCode() !== 0) {
return self::establishNewMultiplexedConnection($server);
}
// Connection exists, ensure we have metadata for age tracking
if (self::getConnectionAge($server) === null) {
// Existing connection but no metadata, store current time as fallback
self::storeConnectionMetadata($server);
}
// Connection exists, check if it needs refresh due to age
if (self::isConnectionExpired($server)) {
return self::refreshMultiplexedConnection($server);
}
// Perform health check if enabled
if (config('constants.ssh.mux_health_check_enabled')) {
if (! self::isConnectionHealthy($server)) {
return self::refreshMultiplexedConnection($server);
}
}
return true;
return self::isMultiplexingEnabled();
}
public static function establishNewMultiplexedConnection(Server $server): bool
public static function removeMuxFile(Server $server): void
{
$sshConfig = self::serverSshConfiguration($server);
$sshKeyLocation = $sshConfig['sshKeyLocation'];
$muxSocket = $sshConfig['muxFilename'];
$connectionTimeout = self::getConnectionTimeout($server);
$serverInterval = config('constants.ssh.server_interval');
$muxPersistTime = config('constants.ssh.mux_persist_time');
$establishCommand = "ssh -fNM -o ControlMaster=auto -o ControlPath=$muxSocket -o ControlPersist={$muxPersistTime} ";
if (data_get($server, 'settings.is_cloudflare_tunnel')) {
$establishCommand .= ' -o ProxyCommand="cloudflared access ssh --hostname %h" ';
}
$establishCommand .= self::getCommonSshOptions($server, $sshKeyLocation, $connectionTimeout, $serverInterval);
$establishCommand .= self::escapedUserAtHost($server);
$establishProcess = Process::run($establishCommand);
if ($establishProcess->exitCode() !== 0) {
return false;
}
// Store connection metadata for tracking
self::storeConnectionMetadata($server);
return true;
}
public static function removeMuxFile(Server $server)
{
$sshConfig = self::serverSshConfiguration($server);
$muxSocket = $sshConfig['muxFilename'];
$closeCommand = "ssh -O exit -o ControlPath=$muxSocket ";
if (data_get($server, 'settings.is_cloudflare_tunnel')) {
$closeCommand .= '-o ProxyCommand="cloudflared access ssh --hostname %h" ';
}
$closeCommand .= self::escapedUserAtHost($server);
$closeCommand = self::muxControlCommand($server, 'exit');
Process::run($closeCommand);
// Clear connection metadata from cache
self::clearConnectionMetadata($server);
}
public static function generateScpCommand(Server $server, string $source, string $dest)
private static function muxControlCommand(Server $server, string $operation): string
{
$command = "ssh -O {$operation} -o ControlPath=".self::muxSocket($server).' ';
if (data_get($server, 'settings.is_cloudflare_tunnel')) {
$command .= '-o ProxyCommand="cloudflared access ssh --hostname %h" ';
}
return $command.self::escapedUserAtHost($server);
}
public static function generateScpCommand(Server $server, string $source, string $dest): string
{
$sshConfig = self::serverSshConfiguration($server);
$sshKeyLocation = $sshConfig['sshKeyLocation'];
$muxSocket = $sshConfig['muxFilename'];
$scpCommand = 'timeout '.config('constants.ssh.command_timeout').' scp ';
$timeout = config('constants.ssh.command_timeout');
$muxPersistTime = config('constants.ssh.mux_persist_time');
$scp_command = "timeout $timeout scp ";
if ($server->isIpv6()) {
$scp_command .= '-6 ';
$scpCommand .= '-6 ';
}
if (self::isMultiplexingEnabled()) {
try {
if (self::ensureMultiplexedConnection($server)) {
$scp_command .= "-o ControlMaster=auto -o ControlPath=$muxSocket -o ControlPersist={$muxPersistTime} ";
}
} catch (\Exception $e) {
Log::warning('SSH multiplexing failed for SCP, falling back to non-multiplexed connection', [
'server' => $server->name ?? $server->ip,
'error' => $e->getMessage(),
]);
// Continue without multiplexing
}
$scpCommand .= self::multiplexingOptions($server);
}
if (data_get($server, 'settings.is_cloudflare_tunnel')) {
$scp_command .= '-o ProxyCommand="cloudflared access ssh --hostname %h" ';
$scpCommand .= '-o ProxyCommand="cloudflared access ssh --hostname %h" ';
}
$scp_command .= self::getCommonSshOptions($server, $sshKeyLocation, self::getConnectionTimeout($server), config('constants.ssh.server_interval'), isScp: true);
$scpCommand .= self::getCommonSshOptions($server, $sshKeyLocation, self::getConnectionTimeout($server), config('constants.ssh.server_interval'), isScp: true);
if ($server->isIpv6()) {
$scp_command .= "{$source} ".escapeshellarg($server->user).'@['.escapeshellarg($server->ip)."]:{$dest}";
} else {
$scp_command .= "{$source} ".self::escapedUserAtHost($server).":{$dest}";
return $scpCommand."{$source} ".escapeshellarg($server->user).'@['.escapeshellarg($server->ip)."]:{$dest}";
}
return $scp_command;
return $scpCommand."{$source} ".self::escapedUserAtHost($server).":{$dest}";
}
public static function generateSshCommand(Server $server, string $command, bool $disableMultiplexing = false)
public static function generateSshCommand(Server $server, string $command, bool $disableMultiplexing = false): string
{
if ($server->settings->force_disabled) {
throw new \RuntimeException('Server is disabled.');
@@ -161,40 +80,36 @@ class SshMultiplexingHelper
self::validateSshKey($server->privateKey);
$muxSocket = $sshConfig['muxFilename'];
$sshCommand = 'timeout '.config('constants.ssh.command_timeout').' ssh ';
$timeout = config('constants.ssh.command_timeout');
$muxPersistTime = config('constants.ssh.mux_persist_time');
$ssh_command = "timeout $timeout ssh ";
$multiplexingSuccessful = false;
if (! $disableMultiplexing && self::isMultiplexingEnabled()) {
try {
$multiplexingSuccessful = self::ensureMultiplexedConnection($server);
if ($multiplexingSuccessful) {
$ssh_command .= "-o ControlMaster=auto -o ControlPath=$muxSocket -o ControlPersist={$muxPersistTime} ";
}
} catch (\Exception $e) {
// Continue without multiplexing
}
$sshCommand .= self::multiplexingOptions($server);
}
if (data_get($server, 'settings.is_cloudflare_tunnel')) {
$ssh_command .= "-o ProxyCommand='cloudflared access ssh --hostname %h' ";
$sshCommand .= "-o ProxyCommand='cloudflared access ssh --hostname %h' ";
}
$ssh_command .= self::getCommonSshOptions($server, $sshKeyLocation, self::getConnectionTimeout($server), config('constants.ssh.server_interval'));
$sshCommand .= self::getCommonSshOptions($server, $sshKeyLocation, self::getConnectionTimeout($server), config('constants.ssh.server_interval'));
$delimiter = Hash::make($command);
$delimiter = base64_encode($delimiter);
$delimiter = base64_encode(Hash::make($command));
$command = str_replace($delimiter, '', $command);
$ssh_command .= self::escapedUserAtHost($server)." 'bash -se' << \\$delimiter".PHP_EOL
return $sshCommand.self::escapedUserAtHost($server)." 'bash -se' << \\$delimiter".PHP_EOL
.$command.PHP_EOL
.$delimiter;
}
return $ssh_command;
private static function multiplexingOptions(Server $server): string
{
return '-o ControlMaster=auto '
.'-o ControlPath='.self::muxSocket($server).' '
.'-o ControlPersist='.config('constants.ssh.mux_persist_time').' ';
}
private static function muxSocket(Server $server): string
{
return '/var/www/html/storage/app/ssh/mux/mux_'.$server->uuid;
}
private static function escapedUserAtHost(Server $server): string
@@ -231,7 +146,6 @@ class SshMultiplexingHelper
$privateKey->storeInFileSystem();
}
// Ensure correct permissions (SSH requires 0600)
if (file_exists($keyLocation)) {
$currentPerms = fileperms($keyLocation) & 0777;
if ($currentPerms !== 0600 && ! chmod($keyLocation, 0600)) {
@@ -262,90 +176,10 @@ class SshMultiplexingHelper
.'-o RequestTTY=no '
.'-o LogLevel=ERROR ';
// Bruh
if ($isScp) {
$options .= '-P '.escapeshellarg((string) $server->port).' ';
} else {
$options .= '-p '.escapeshellarg((string) $server->port).' ';
return $options.'-P '.escapeshellarg((string) $server->port).' ';
}
return $options;
}
/**
* Check if the multiplexed connection is healthy by running a test command
*/
public static function isConnectionHealthy(Server $server): bool
{
$sshConfig = self::serverSshConfiguration($server);
$muxSocket = $sshConfig['muxFilename'];
$healthCheckTimeout = config('constants.ssh.mux_health_check_timeout');
$healthCommand = "timeout $healthCheckTimeout ssh -o ControlMaster=auto -o ControlPath=$muxSocket ";
if (data_get($server, 'settings.is_cloudflare_tunnel')) {
$healthCommand .= '-o ProxyCommand="cloudflared access ssh --hostname %h" ';
}
$healthCommand .= self::escapedUserAtHost($server)." 'echo \"health_check_ok\"'";
$process = Process::run($healthCommand);
$isHealthy = $process->exitCode() === 0 && str_contains($process->output(), 'health_check_ok');
return $isHealthy;
}
/**
* Check if the connection has exceeded its maximum age
*/
public static function isConnectionExpired(Server $server): bool
{
$connectionAge = self::getConnectionAge($server);
$maxAge = config('constants.ssh.mux_max_age');
return $connectionAge !== null && $connectionAge > $maxAge;
}
/**
* Get the age of the current connection in seconds
*/
public static function getConnectionAge(Server $server): ?int
{
$cacheKey = "ssh_mux_connection_time_{$server->uuid}";
$connectionTime = Cache::get($cacheKey);
if ($connectionTime === null) {
return null;
}
return time() - $connectionTime;
}
/**
* Refresh a multiplexed connection by closing and re-establishing it
*/
public static function refreshMultiplexedConnection(Server $server): bool
{
// Close existing connection
self::removeMuxFile($server);
// Establish new connection
return self::establishNewMultiplexedConnection($server);
}
/**
* Store connection metadata when a new connection is established
*/
private static function storeConnectionMetadata(Server $server): void
{
$cacheKey = "ssh_mux_connection_time_{$server->uuid}";
Cache::put($cacheKey, time(), config('constants.ssh.mux_persist_time') + 300); // Cache slightly longer than persist time
}
/**
* Clear connection metadata from cache
*/
private static function clearConnectionMetadata(Server $server): void
{
$cacheKey = "ssh_mux_connection_time_{$server->uuid}";
Cache::forget($cacheKey);
return $options.'-p '.escapeshellarg((string) $server->port).' ';
}
}
app/Http/Controllers/Api/SentinelController.php
+146
View File
@@ -0,0 +1,146 @@
<?php
namespace App\Http\Controllers\Api;
use App\Http\Controllers\Controller;
use App\Jobs\PushServerUpdateJob;
use App\Models\Server;
use Exception;
use Illuminate\Http\Request;
use Illuminate\Support\Facades\Cache;
class SentinelController extends Controller
{
/**
* Handle a Sentinel agent metrics push.
*
* Sentinel pushes its full container list on a fixed interval (default 60s),
* even when nothing changed. To avoid dispatching one PushServerUpdateJob per
* server per minute, the job is only dispatched when the container state hash
* changes, or when the force window has elapsed.
*/
public function push(Request $request)
{
$token = $request->header('Authorization');
if (! $token) {
auditLogWebhookFailure('sentinel', 'token_missing');
return response()->json(['message' => 'Unauthorized'], 401);
}
$naked_token = str_replace('Bearer ', '', $token);
try {
$decrypted = decrypt($naked_token);
$decrypted_token = json_decode($decrypted, true);
} catch (Exception $e) {
auditLogWebhookFailure('sentinel', 'decrypt_failed');
return response()->json(['message' => 'Invalid token'], 401);
}
$server_uuid = data_get($decrypted_token, 'server_uuid');
if (! $server_uuid) {
auditLogWebhookFailure('sentinel', 'invalid_token_payload');
return response()->json(['message' => 'Invalid token'], 401);
}
$server = Server::where('uuid', $server_uuid)->first();
if (! $server) {
auditLogWebhookFailure('sentinel', 'server_not_found', [
'server_uuid' => $server_uuid,
]);
return response()->json(['message' => 'Server not found'], 404);
}
if (isCloud() && data_get($server->team->subscription, 'stripe_invoice_paid', false) === false && $server->team->id !== 0) {
auditLogWebhookFailure('sentinel', 'subscription_unpaid', [
'server_uuid' => $server->uuid,
'team_id' => $server->team_id,
]);
return response()->json(['message' => 'Unauthorized'], 401);
}
if ($server->isFunctional() === false) {
auditLogWebhookFailure('sentinel', 'server_not_functional', [
'server_uuid' => $server->uuid,
'team_id' => $server->team_id,
]);
return response()->json(['message' => 'Server is not functional'], 401);
}
if ($server->settings->sentinel_token !== $naked_token) {
auditLogWebhookFailure('sentinel', 'token_mismatch', [
'server_uuid' => $server->uuid,
'team_id' => $server->team_id,
]);
return response()->json(['message' => 'Unauthorized'], 401);
}
$data = $request->all();
// Heartbeat MUST update on every push — drives isSentinelLive() and SSH-check skipping.
$server->sentinelHeartbeat();
if ($this->shouldDispatchUpdate($server, $data)) {
PushServerUpdateJob::dispatch($server, $data);
}
auditLog('sentinel.metrics_pushed', [
'server_uuid' => $server->uuid,
'team_id' => $server->team_id,
]);
return response()->json(['message' => 'ok'], 200);
}
/**
* Decide whether PushServerUpdateJob should be dispatched for this push.
*
* Dispatches when: first push (no cached hash), the container state changed,
* or the force window elapsed.
*/
private function shouldDispatchUpdate(Server $server, array $data): bool
{
$hash = $this->containerStateHash($data);
$hashKey = "sentinel:push-hash:{$server->id}";
$forceKey = "sentinel:push-force:{$server->id}";
$cachedHash = Cache::get($hashKey);
$forceActive = Cache::has($forceKey);
$shouldDispatch = $cachedHash === null || $cachedHash !== $hash || ! $forceActive;
if ($shouldDispatch) {
// Day-long TTL bounds memory if a server stops pushing entirely.
Cache::put($hashKey, $hash, now()->addDay());
Cache::put($forceKey, true, config('constants.sentinel.push_force_interval_seconds', 300));
}
return $shouldDispatch;
}
/**
* Build a stable hash of container state.
*
* Covers [name, state, health_status] only — metrics and
* filesystem_usage_root are excluded on purpose (disk % churns constantly
* and would defeat the hash; the storage check is separately cache-gated
* inside PushServerUpdateJob). Sorted by name so container ordering from
* Sentinel does not affect the hash.
*/
private function containerStateHash(array $data): string
{
$containers = collect(data_get($data, 'containers', []))
->map(fn ($c) => [
'name' => data_get($c, 'name'),
'state' => data_get($c, 'state'),
'health_status' => data_get($c, 'health_status'),
])
->sortBy('name')
->values()
->all();
return hash('xxh128', json_encode($containers));
}
}
app/Http/Controllers/Webhook/Bitbucket.php
+13 -17
View File
@@ -5,6 +5,7 @@ namespace App\Http\Controllers\Webhook;
use App\Actions\Application\CleanupPreviewDeployment;
use App\Http\Controllers\Controller;
use App\Http\Controllers\Webhook\Concerns\DetectsSkipDeployCommits;
use App\Http\Controllers\Webhook\Concerns\MatchesManualWebhookApplications;
use App\Models\Application;
use App\Models\ApplicationPreview;
use Exception;
@@ -14,6 +15,7 @@ use Visus\Cuid2\Cuid2;
class Bitbucket extends Controller
{
use DetectsSkipDeployCommits;
use MatchesManualWebhookApplications;
public function manual(Request $request)
{
@@ -62,8 +64,14 @@ class Bitbucket extends Controller
$skip_deploy_pr = self::shouldSkipDeployAny([$pull_request_title]);
$commit = data_get($payload, 'pullrequest.source.commit.hash');
}
$applications = Application::where('git_repository', 'like', "%$full_name%");
$applications = $applications->where('git_branch', $branch)->get();
$full_name = $this->manualWebhookRepositoryFullName($full_name);
if ($full_name === null) {
return response([
'status' => 'failed',
'message' => 'Nothing to do. Invalid repository.',
]);
}
$applications = $this->manualWebhookApplications(Application::query()->where('git_branch', $branch), $full_name);
if ($applications->isEmpty()) {
return response([
'status' => 'failed',
@@ -79,11 +87,7 @@ class Bitbucket extends Controller
'repository' => $full_name ?? null,
'event' => $x_bitbucket_event,
]);
$return_payloads->push([
'application' => $application->name,
'status' => 'failed',
'message' => 'Webhook secret not configured.',
]);
$return_payloads->push($this->unauthenticatedManualWebhookFailurePayload());
continue;
}
@@ -97,11 +101,7 @@ class Bitbucket extends Controller
'repository' => $full_name ?? null,
'event' => $x_bitbucket_event,
]);
$return_payloads->push([
'application' => $application->name,
'status' => 'failed',
'message' => 'Invalid signature.',
]);
$return_payloads->push($this->unauthenticatedManualWebhookFailurePayload());
continue;
}
@@ -114,11 +114,7 @@ class Bitbucket extends Controller
'repository' => $full_name ?? null,
'event' => $x_bitbucket_event,
]);
$return_payloads->push([
'application' => $application->name,
'status' => 'failed',
'message' => 'Invalid signature.',
]);
$return_payloads->push($this->unauthenticatedManualWebhookFailurePayload());
continue;
}
app/Http/Controllers/Webhook/Concerns/MatchesManualWebhookApplications.php
+104
View File
@@ -0,0 +1,104 @@
<?php
namespace App\Http\Controllers\Webhook\Concerns;
use App\Models\Application;
use Illuminate\Database\Eloquent\Builder;
use Illuminate\Support\Collection;
use Illuminate\Support\Str;
trait MatchesManualWebhookApplications
{
protected function manualWebhookRepositoryFullName(mixed $fullName): ?string
{
if (! is_string($fullName)) {
return null;
}
$fullName = trim($fullName, " \t\n\r\0\x0B/");
if ($fullName === '') {
return null;
}
if (! preg_match('/\A[A-Za-z0-9_.-]+(?:\/[A-Za-z0-9_.-]+)+\z/', $fullName)) {
return null;
}
return $this->normalizeManualWebhookRepositoryPath($fullName);
}
/**
* @return Collection<int, Application>
*/
protected function manualWebhookApplications(Builder $query, string $fullName): Collection
{
return $query->get()
->filter(fn (Application $application): bool => $this->manualWebhookRepositoryMatches($application->git_repository, $fullName))
->values();
}
protected function manualWebhookRepositoryMatches(?string $gitRepository, string $fullName): bool
{
$repositoryPath = $this->canonicalManualWebhookRepository($gitRepository);
if ($repositoryPath === null) {
return false;
}
// Git hosts (GitHub, GitLab, Gitea, Bitbucket) treat owner/repo names
// case-insensitively, so compare the canonical paths case-insensitively.
return hash_equals(mb_strtolower($fullName), mb_strtolower($repositoryPath));
}
/**
* @return array{status: string, message: string}
*/
protected function unauthenticatedManualWebhookFailurePayload(): array
{
return [
'status' => 'failed',
'message' => 'Invalid signature.',
];
}
protected function canonicalManualWebhookRepository(?string $gitRepository): ?string
{
if (! is_string($gitRepository)) {
return null;
}
$gitRepository = trim($gitRepository);
if ($gitRepository === '') {
return null;
}
$path = null;
$parts = parse_url($gitRepository);
if (is_array($parts) && isset($parts['scheme'])) {
$path = data_get($parts, 'path');
} elseif (Str::startsWith($gitRepository, 'git@') && str_contains($gitRepository, ':')) {
$path = Str::after($gitRepository, ':');
} else {
$path = $gitRepository;
}
if (! is_string($path) || $path === '') {
return null;
}
return $this->normalizeManualWebhookRepositoryPath($path);
}
protected function normalizeManualWebhookRepositoryPath(string $path): string
{
$path = trim($path);
$path = strtok($path, '?#') ?: $path;
$path = trim($path, '/');
$path = preg_replace('/\.git\z/i', '', $path) ?? $path;
return $path;
}
}
app/Http/Controllers/Webhook/Gitea.php
+11 -13
View File
@@ -5,6 +5,7 @@ namespace App\Http\Controllers\Webhook;
use App\Actions\Application\CleanupPreviewDeployment;
use App\Http\Controllers\Controller;
use App\Http\Controllers\Webhook\Concerns\DetectsSkipDeployCommits;
use App\Http\Controllers\Webhook\Concerns\MatchesManualWebhookApplications;
use App\Models\Application;
use App\Models\ApplicationPreview;
use Exception;
@@ -15,6 +16,7 @@ use Visus\Cuid2\Cuid2;
class Gitea extends Controller
{
use DetectsSkipDeployCommits;
use MatchesManualWebhookApplications;
public function manual(Request $request)
{
@@ -58,15 +60,19 @@ class Gitea extends Controller
if (! $branch) {
return response('Nothing to do. No branch found in the request.');
}
$applications = Application::where('git_repository', 'like', "%$full_name%");
$full_name = $this->manualWebhookRepositoryFullName($full_name);
if ($full_name === null) {
return response('Nothing to do. Invalid repository.');
}
$applications = Application::query();
if ($x_gitea_event === 'push') {
$applications = $applications->where('git_branch', $branch)->get();
$applications = $this->manualWebhookApplications($applications->where('git_branch', $branch), $full_name);
if ($applications->isEmpty()) {
return response("Nothing to do. No applications found with deploy key set, branch is '$branch' and Git Repository name has $full_name.");
}
}
if ($x_gitea_event === 'pull_request') {
$applications = $applications->where('git_branch', $base_branch)->get();
$applications = $this->manualWebhookApplications($applications->where('git_branch', $base_branch), $full_name);
if ($applications->isEmpty()) {
return response("Nothing to do. No applications found with branch '$base_branch'.");
}
@@ -80,11 +86,7 @@ class Gitea extends Controller
'repository' => $full_name ?? null,
'event' => $x_gitea_event,
]);
$return_payloads->push([
'application' => $application->name,
'status' => 'failed',
'message' => 'Webhook secret not configured.',
]);
$return_payloads->push($this->unauthenticatedManualWebhookFailurePayload());
continue;
}
@@ -96,11 +98,7 @@ class Gitea extends Controller
'repository' => $full_name ?? null,
'event' => $x_gitea_event,
]);
$return_payloads->push([
'application' => $application->name,
'status' => 'failed',
'message' => 'Invalid signature.',
]);
$return_payloads->push($this->unauthenticatedManualWebhookFailurePayload());
continue;
}
app/Http/Controllers/Webhook/Github.php
+135 -53
View File
@@ -4,6 +4,7 @@ namespace App\Http\Controllers\Webhook;
use App\Http\Controllers\Controller;
use App\Http\Controllers\Webhook\Concerns\DetectsSkipDeployCommits;
use App\Http\Controllers\Webhook\Concerns\MatchesManualWebhookApplications;
use App\Jobs\GithubAppPermissionJob;
use App\Jobs\ProcessGithubPullRequestWebhook;
use App\Models\Application;
@@ -11,6 +12,7 @@ use App\Models\GithubApp;
use App\Models\PrivateKey;
use Exception;
use Illuminate\Http\Request;
use Illuminate\Support\Facades\Cache;
use Illuminate\Support\Facades\Http;
use Illuminate\Support\Str;
use Visus\Cuid2\Cuid2;
@@ -18,6 +20,7 @@ use Visus\Cuid2\Cuid2;
class Github extends Controller
{
use DetectsSkipDeployCommits;
use MatchesManualWebhookApplications;
public function manual(Request $request)
{
@@ -66,15 +69,19 @@ class Github extends Controller
if (! $branch) {
return response('Nothing to do. No branch found in the request.');
}
$applications = Application::where('git_repository', 'like', "%$full_name%");
$full_name = $this->manualWebhookRepositoryFullName($full_name);
if ($full_name === null) {
return response('Nothing to do. Invalid repository.');
}
$applications = Application::query();
if ($x_github_event === 'push') {
$applications = $applications->where('git_branch', $branch)->get();
$applications = $this->manualWebhookApplications($applications->where('git_branch', $branch), $full_name);
if ($applications->isEmpty()) {
return response("Nothing to do. No applications found with deploy key set, branch is '$branch' and Git Repository name has $full_name.");
}
}
if ($x_github_event === 'pull_request') {
$applications = $applications->where('git_branch', $base_branch)->get();
$applications = $this->manualWebhookApplications($applications->where('git_branch', $base_branch), $full_name);
if ($applications->isEmpty()) {
return response("Nothing to do. No applications found for repo $full_name and branch '$base_branch'.");
}
@@ -93,11 +100,7 @@ class Github extends Controller
'repository' => $full_name ?? null,
'mode' => 'manual',
]);
$return_payloads->push([
'application' => $application->name,
'status' => 'failed',
'message' => 'Webhook secret not configured.',
]);
$return_payloads->push($this->unauthenticatedManualWebhookFailurePayload());
continue;
}
@@ -109,11 +112,7 @@ class Github extends Controller
'repository' => $full_name ?? null,
'mode' => 'manual',
]);
$return_payloads->push([
'application' => $application->name,
'status' => 'failed',
'message' => 'Invalid signature.',
]);
$return_payloads->push($this->unauthenticatedManualWebhookFailurePayload());
continue;
}
@@ -454,53 +453,136 @@ class Github extends Controller
public function redirect(Request $request)
{
try {
$code = $request->get('code');
$state = $request->get('state');
$github_app = GithubApp::where('uuid', $state)->firstOrFail();
$api_url = data_get($github_app, 'api_url');
$data = Http::withBody(null)->accept('application/vnd.github+json')->post("$api_url/app-manifests/$code/conversions")->throw()->json();
$id = data_get($data, 'id');
$slug = data_get($data, 'slug');
$client_id = data_get($data, 'client_id');
$client_secret = data_get($data, 'client_secret');
$private_key = data_get($data, 'pem');
$webhook_secret = data_get($data, 'webhook_secret');
$private_key = PrivateKey::create([
'name' => "github-app-{$slug}",
'private_key' => $private_key,
'team_id' => $github_app->team_id,
'is_git_related' => true,
]);
$github_app->name = $slug;
$github_app->app_id = $id;
$github_app->client_id = $client_id;
$github_app->client_secret = $client_secret;
$github_app->webhook_secret = $webhook_secret;
$github_app->private_key_id = $private_key->id;
$github_app->save();
$code = (string) $request->query('code', '');
abort_if(blank($code), 422, 'Missing GitHub App manifest code.');
return redirect()->route('source.github.show', ['github_app_uuid' => $github_app->uuid]);
} catch (Exception $e) {
return handleError($e);
}
$github_app = $this->consumeGithubAppSetupState(
request: $request,
state: (string) $request->query('state', ''),
action: 'manifest',
);
abort_if($this->githubAppHasManifestCredentials($github_app), 403, 'GitHub App credentials are already configured.');
$api_url = data_get($github_app, 'api_url');
$data = Http::withBody(null)
->accept('application/vnd.github+json')
->timeout(10)
->connectTimeout(5)
->post("$api_url/app-manifests/$code/conversions")
->throw()
->json();
$id = data_get($data, 'id');
$slug = data_get($data, 'slug');
$client_id = data_get($data, 'client_id');
$client_secret = data_get($data, 'client_secret');
$private_key = data_get($data, 'pem');
$webhook_secret = data_get($data, 'webhook_secret');
abort_if(blank($id) || blank($slug) || blank($client_id) || blank($client_secret) || blank($private_key) || blank($webhook_secret), 422, 'GitHub App manifest conversion response is incomplete.');
$private_key = PrivateKey::create([
'name' => "github-app-{$slug}",
'private_key' => $private_key,
'team_id' => $github_app->team_id,
'is_git_related' => true,
]);
$github_app->name = $slug;
$github_app->app_id = $id;
$github_app->client_id = $client_id;
$github_app->client_secret = $client_secret;
$github_app->webhook_secret = $webhook_secret;
$github_app->private_key_id = $private_key->id;
$github_app->save();
return redirect()->route('source.github.show', ['github_app_uuid' => $github_app->uuid]);
}
public function install(Request $request)
{
try {
$installation_id = $request->get('installation_id');
$source = $request->get('source');
$setup_action = $request->get('setup_action');
$github_app = GithubApp::where('uuid', $source)->firstOrFail();
if ($setup_action === 'install') {
$github_app->installation_id = $installation_id;
$github_app->save();
}
$source = (string) $request->query('source', '');
abort_if(blank($source), 404);
$github_app = GithubApp::ownedByCurrentTeam()->where('uuid', $source)->firstOrFail();
$setup_action = (string) $request->query('setup_action', '');
if ($setup_action !== 'install') {
return redirect()->route('source.github.show', ['github_app_uuid' => $github_app->uuid]);
} catch (Exception $e) {
return handleError($e);
}
$installation_id = (string) $request->query('installation_id', '');
abort_unless(ctype_digit($installation_id), 422, 'Missing GitHub App installation id.');
abort_unless(
$this->githubInstallationBelongsToApp($github_app, $installation_id),
403,
'GitHub App installation could not be verified.'
);
$github_app->installation_id = $installation_id;
$github_app->save();
return redirect()->route('source.github.show', ['github_app_uuid' => $github_app->uuid]);
}
/**
* Verify that the given installation id actually belongs to this GitHub App.
*
* The installation id arrives as an untrusted query parameter on an
* unauthenticated-reachable GET callback, so it must be confirmed against
* the GitHub API using the App's own credentials before it is persisted.
*/
private function githubInstallationBelongsToApp(GithubApp $github_app, string $installation_id): bool
{
if (blank($github_app->app_id) || blank($github_app->privateKey?->private_key)) {
return false;
}
try {
$jwt = generateGithubJwt($github_app);
$response = Http::withHeaders([
'Authorization' => "Bearer $jwt",
'Accept' => 'application/vnd.github+json',
])
->timeout(10)
->connectTimeout(5)
->get("{$github_app->api_url}/app/installations/{$installation_id}");
return $response->successful()
&& (string) data_get($response->json(), 'app_id') === (string) $github_app->app_id;
} catch (\Throwable) {
return false;
}
}
private function consumeGithubAppSetupState(Request $request, string $state, string $action): GithubApp
{
abort_if(blank($state), 404);
$payload = Cache::pull($this->githubAppSetupStateCacheKey($state));
abort_unless(is_array($payload), 404);
abort_unless(data_get($payload, 'action') === $action, 404);
$team_id = $request->user()?->currentTeam()?->id;
abort_unless(! is_null($team_id) && (int) data_get($payload, 'team_id') === $team_id, 403);
return GithubApp::whereKey(data_get($payload, 'github_app_id'))
->where('team_id', data_get($payload, 'team_id'))
->firstOrFail();
}
private function githubAppSetupStateCacheKey(string $state): string
{
return 'github-app-setup-state:'.hash('sha256', $state);
}
private function githubAppHasManifestCredentials(GithubApp $github_app): bool
{
return filled($github_app->app_id)
|| filled($github_app->client_id)
|| filled($github_app->client_secret)
|| filled($github_app->webhook_secret)
|| filled($github_app->private_key_id);
}
}
app/Http/Controllers/Webhook/Gitlab.php
+16 -13
View File
@@ -5,6 +5,7 @@ namespace App\Http\Controllers\Webhook;
use App\Actions\Application\CleanupPreviewDeployment;
use App\Http\Controllers\Controller;
use App\Http\Controllers\Webhook\Concerns\DetectsSkipDeployCommits;
use App\Http\Controllers\Webhook\Concerns\MatchesManualWebhookApplications;
use App\Models\Application;
use App\Models\ApplicationPreview;
use Exception;
@@ -15,6 +16,7 @@ use Visus\Cuid2\Cuid2;
class Gitlab extends Controller
{
use DetectsSkipDeployCommits;
use MatchesManualWebhookApplications;
public function manual(Request $request)
{
@@ -85,9 +87,18 @@ class Gitlab extends Controller
return response($return_payloads);
}
}
$applications = Application::where('git_repository', 'like', "%$full_name%");
$full_name = $this->manualWebhookRepositoryFullName($full_name);
if ($full_name === null) {
$return_payloads->push([
'status' => 'failed',
'message' => 'Nothing to do. Invalid repository.',
]);
return response($return_payloads);
}
$applications = Application::query();
if ($x_gitlab_event === 'push') {
$applications = $applications->where('git_branch', $branch)->get();
$applications = $this->manualWebhookApplications($applications->where('git_branch', $branch), $full_name);
if ($applications->isEmpty()) {
$return_payloads->push([
'status' => 'failed',
@@ -98,7 +109,7 @@ class Gitlab extends Controller
}
}
if ($x_gitlab_event === 'merge_request') {
$applications = $applications->where('git_branch', $base_branch)->get();
$applications = $this->manualWebhookApplications($applications->where('git_branch', $base_branch), $full_name);
if ($applications->isEmpty()) {
$return_payloads->push([
'status' => 'failed',
@@ -117,11 +128,7 @@ class Gitlab extends Controller
'repository' => $full_name ?? null,
'event' => $x_gitlab_event,
]);
$return_payloads->push([
'application' => $application->name,
'status' => 'failed',
'message' => 'Webhook secret not configured.',
]);
$return_payloads->push($this->unauthenticatedManualWebhookFailurePayload());
continue;
}
@@ -132,11 +139,7 @@ class Gitlab extends Controller
'repository' => $full_name ?? null,
'event' => $x_gitlab_event,
]);
$return_payloads->push([
'application' => $application->name,
'status' => 'failed',
'message' => 'Invalid signature.',
]);
$return_payloads->push($this->unauthenticatedManualWebhookFailurePayload());
continue;
}
app/Jobs/ApplicationDeploymentJob.php
+1 -1
View File
@@ -197,7 +197,7 @@ class ApplicationDeploymentJob implements ShouldBeEncrypted, ShouldQueue
public function __construct(public int $application_deployment_queue_id)
{
$this->onQueue('high');
$this->onQueue(deployment_queue());
$this->application_deployment_queue = ApplicationDeploymentQueue::find($this->application_deployment_queue_id);
$this->nixpacks_plan_json = collect([]);
app/Jobs/CleanupStaleMultiplexedConnections.php
+218 -5
View File
@@ -9,6 +9,7 @@ use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Foundation\Bus\Dispatchable;
use Illuminate\Queue\InteractsWithQueue;
use Illuminate\Queue\SerializesModels;
use Illuminate\Support\Facades\Log;
use Illuminate\Support\Facades\Process;
use Illuminate\Support\Facades\Storage;
@@ -20,6 +21,199 @@ class CleanupStaleMultiplexedConnections implements ShouldQueue
{
$this->cleanupStaleConnections();
$this->cleanupNonExistentServerConnections();
$this->cleanupDuplicateSshProcesses();
$this->cleanupOrphanedSshProcesses();
$this->cleanupOrphanedCloudflaredProcesses();
}
/**
* Once two background ssh masters share the same ControlPath, OpenSSH's
* control socket state is no longer trustworthy: `ssh -O check` may report
* one PID while the socket lifecycle is tied to another. Reset the whole
* duplicate group rather than trying to choose an owner.
*/
private function cleanupDuplicateSshProcesses(): void
{
$muxDir = storage_path('app/ssh/mux');
$groups = [];
foreach ($this->listProcesses() as $process) {
$controlPath = $this->extractControlPath($process['args']);
if (! is_string($controlPath) || ! str_starts_with($controlPath, $muxDir.'/')) {
continue;
}
$groups[$controlPath][] = $process;
}
foreach ($groups as $controlPath => $processes) {
if (count($processes) < 2) {
continue;
}
$this->resetDuplicateGroup($controlPath, $processes);
}
}
/**
* Kill backgrounded ssh master processes that lost the ControlPath socket
* race. Such processes are not masters, so ControlPersist never reaps them
* and they leak memory until the container restarts. A legitimate master
* always owns its socket file; an orphan has none.
*
* Processes younger than the minimum age are skipped: a freshly forked
* master creates its socket a few milliseconds after starting, so a young
* process with no socket may simply be mid-establish rather than orphaned.
*/
private function cleanupOrphanedSshProcesses(): void
{
$muxDir = storage_path('app/ssh/mux');
$minAge = (int) config('constants.ssh.mux_orphan_min_age');
foreach ($this->listProcesses() as $process) {
// Only ever touch ssh processes pointing at Coolify's mux directory.
$controlPath = $this->extractControlPath($process['args']);
if (! is_string($controlPath) || ! str_starts_with($controlPath, $muxDir.'/')) {
continue;
}
if ($process['etimes'] >= $minAge && ! file_exists($controlPath)) {
$this->reapOrphan('ssh', $process);
}
}
}
/**
* Kill orphaned `cloudflared access ssh` proxy processes. Each is spawned
* as the SSH ProxyCommand transport for a Cloudflare Tunnel server and must
* die with its parent ssh. When that ssh is killed or orphaned (e.g. a lost
* mux master), the cloudflared process can leak and accumulate. A legitimate
* proxy always has a live ssh parent; one without is safe to reap.
*
* Processes younger than the minimum age are skipped so a proxy whose parent
* ssh is still starting up, or a transient `ssh -O check` proxy mid-exit, is
* never mistaken for an orphan.
*/
private function cleanupOrphanedCloudflaredProcesses(): void
{
$minAge = (int) config('constants.ssh.mux_orphan_min_age');
$processes = $this->listProcesses();
$sshPids = [];
foreach ($processes as $process) {
// The ssh binary itself, not `cloudflared access ssh` (space before ssh).
if (preg_match('#(^|/)ssh\s#', $process['args'])) {
$sshPids[$process['pid']] = true;
}
}
foreach ($processes as $process) {
// `cloudflared access ssh`, never the `cloudflared tunnel` daemon.
if (! str_contains($process['args'], 'cloudflared access ssh')) {
continue;
}
// Orphaned when no live ssh process is its parent.
if ($process['etimes'] >= $minAge && ! isset($sshPids[$process['ppid']])) {
$this->reapOrphan('cloudflared', $process);
}
}
}
/**
* Reap a detected orphan process. When orphan reaping is disabled (the
* default), the orphan is only logged — a dry-run mode that lets operators
* verify what would be killed before enabling it for real.
*
* @param array{pid: string, ppid: string, etimes: int, args: string} $process
*/
private function reapOrphan(string $kind, array $process): void
{
if (! config('constants.ssh.mux_orphan_reap_enabled')) {
Log::info("Orphaned {$kind} process detected (dry-run, not killed)", [
'pid' => $process['pid'],
'etimes' => $process['etimes'],
'command' => $process['args'],
]);
return;
}
Process::run('kill '.escapeshellarg($process['pid']));
Log::info("Killed orphaned {$kind} process", [
'pid' => $process['pid'],
'etimes' => $process['etimes'],
'command' => $process['args'],
]);
}
/**
* Snapshot of running processes.
*
* @return list<array{pid: string, ppid: string, etimes: int, args: string}>
*/
private function listProcesses(): array
{
$ps = Process::run('ps -ww -eo pid=,ppid=,etimes=,args=');
if ($ps->exitCode() !== 0) {
return [];
}
$processes = [];
foreach (explode("\n", trim($ps->output())) as $line) {
if (! preg_match('/^\s*(\d+)\s+(\d+)\s+(\d+)\s+(.*)$/', $line, $matches)) {
continue;
}
$processes[] = [
'pid' => $matches[1],
'ppid' => $matches[2],
'etimes' => (int) $matches[3],
'args' => $matches[4],
];
}
return $processes;
}
/**
* @param list<array{pid: string, ppid: string, etimes: int, args: string}> $processes
*/
private function resetDuplicateGroup(string $controlPath, array $processes): void
{
if (! config('constants.ssh.mux_orphan_reap_enabled')) {
Log::info('Duplicate ssh mux processes detected (dry-run, not killed)', [
'control_path' => $controlPath,
'pids' => array_column($processes, 'pid'),
]);
return;
}
foreach ($processes as $process) {
Process::run('kill '.escapeshellarg($process['pid']));
}
if (file_exists($controlPath)) {
@unlink($controlPath);
}
Log::info('Reset duplicate ssh mux processes', [
'control_path' => $controlPath,
'pids' => array_column($processes, 'pid'),
]);
}
private function extractControlPath(string $args): ?string
{
if (! preg_match('/(?:^|\s)-o\s+ControlPath=(?:"([^"]+)"|\'([^\']+)\'|(\S+))/', $args, $matches)) {
if (preg_match('/^ssh:\s+(\S+)\s+\[mux\]$/', $args, $matches)) {
return $matches[1];
}
return null;
}
return $matches[1] ?: ($matches[2] ?: $matches[3]);
}
private function cleanupStaleConnections()
@@ -31,7 +225,7 @@ class CleanupStaleMultiplexedConnections implements ShouldQueue
$server = Server::where('uuid', $serverUuid)->first();
if (! $server) {
$this->removeMultiplexFile($muxFile);
$this->removeMultiplexFile($muxFile, 'server_not_found');
continue;
}
@@ -41,14 +235,14 @@ class CleanupStaleMultiplexedConnections implements ShouldQueue
$checkProcess = Process::run($checkCommand);
if ($checkProcess->exitCode() !== 0) {
$this->removeMultiplexFile($muxFile);
$this->removeMultiplexFile($muxFile, 'connection_check_failed');
} else {
$muxContent = Storage::disk('ssh-mux')->get($muxFile);
$establishedAt = Carbon::parse(substr($muxContent, 37));
$expirationTime = $establishedAt->addSeconds(config('constants.ssh.mux_persist_time'));
if (Carbon::now()->isAfter($expirationTime)) {
$this->removeMultiplexFile($muxFile);
$this->removeMultiplexFile($muxFile, 'expired');
}
}
}
@@ -62,7 +256,7 @@ class CleanupStaleMultiplexedConnections implements ShouldQueue
foreach ($muxFiles as $muxFile) {
$serverUuid = $this->extractServerUuidFromMuxFile($muxFile);
if (! in_array($serverUuid, $existingServerUuids)) {
$this->removeMultiplexFile($muxFile);
$this->removeMultiplexFile($muxFile, 'server_does_not_exist');
}
}
}
@@ -72,11 +266,30 @@ class CleanupStaleMultiplexedConnections implements ShouldQueue
return substr($muxFile, 4);
}
private function removeMultiplexFile($muxFile)
/**
* Close and delete a stale mux socket file. When orphan reaping is disabled
* (the default), the file is only logged — a dry-run mode that lets operators
* verify what would be removed before enabling it for real.
*/
private function removeMultiplexFile(string $muxFile, string $reason): void
{
if (! config('constants.ssh.mux_orphan_reap_enabled')) {
Log::info('Stale mux file detected (dry-run, not removed)', [
'file' => $muxFile,
'reason' => $reason,
]);
return;
}
$muxSocket = "/var/www/html/storage/app/ssh/mux/{$muxFile}";
$closeCommand = "ssh -O exit -o ControlPath={$muxSocket} localhost 2>/dev/null";
Process::run($closeCommand);
Storage::disk('ssh-mux')->delete($muxFile);
Log::info('Removed stale mux file', [
'file' => $muxFile,
'reason' => $reason,
]);
}
}
app/Jobs/DatabaseBackupJob.php
+1 -1
View File
@@ -77,7 +77,7 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue
public function __construct(public ScheduledDatabaseBackup $backup)
{
$this->onQueue('high');
$this->onQueue(crons_queue());
$this->timeout = $backup->timeout ?? 3600;
}
app/Jobs/PushServerUpdateJob.php
+11 -6
View File
@@ -127,15 +127,20 @@ class PushServerUpdateJob implements ShouldBeEncrypted, ShouldQueue, Silenced
}
$data = collect($this->data);
$this->server->sentinelHeartbeat();
// Heartbeat is updated by SentinelController on every push, before dispatch.
$this->containers = collect(data_get($data, 'containers'));
$filesystemUsageRoot = data_get($data, 'filesystem_usage_root.used_percentage');
// Only dispatch storage check when disk percentage actually changes
// Only dispatch the storage check when disk usage is at/above the notification
// threshold AND the value changed. Below the threshold ServerStorageCheckJob
// has nothing to do (it only sends a HighDiskUsage notification), so dispatching
// it is wasted work — and most servers sit well below the threshold.
$diskThreshold = data_get($this->server, 'settings.server_disk_usage_notification_threshold', 80);
$storageCacheKey = 'storage-check:'.$this->server->id;
$lastPercentage = Cache::get($storageCacheKey);
if ($lastPercentage === null || (string) $lastPercentage !== (string) $filesystemUsageRoot) {
if ($filesystemUsageRoot !== null
&& $filesystemUsageRoot >= $diskThreshold
&& (string) $lastPercentage !== (string) $filesystemUsageRoot) {
Cache::put($storageCacheKey, $filesystemUsageRoot, 600);
ServerStorageCheckJob::dispatch($this->server, $filesystemUsageRoot);
}
@@ -500,11 +505,11 @@ class PushServerUpdateJob implements ShouldBeEncrypted, ShouldQueue, Silenced
} catch (\Throwable $e) {
}
} else {
// Connect proxy to networks periodically (every 10 min) to avoid excessive job dispatches.
// Connect proxy to networks periodically as a safety net to avoid excessive job dispatches.
// On-demand triggers (new network, service deploy) use dispatchSync() and bypass this.
$proxyCacheKey = 'connect-proxy:'.$this->server->id;
if (! Cache::has($proxyCacheKey)) {
Cache::put($proxyCacheKey, true, 600);
Cache::put($proxyCacheKey, true, config('constants.proxy.connect_networks_interval_seconds', 3600));
ConnectProxyToNetworksJob::dispatch($this->server);
}
}
app/Jobs/ScheduledJobManager.php
+1 -11
View File
@@ -37,17 +37,7 @@ class ScheduledJobManager implements ShouldQueue
*/
public function __construct()
{
$this->onQueue($this->determineQueue());
}
private function determineQueue(): string
{
$preferredQueue = 'crons';
$fallbackQueue = 'high';
$configuredQueues = explode(',', env('HORIZON_QUEUES', 'high,default'));
return in_array($preferredQueue, $configuredQueues) ? $preferredQueue : $fallbackQueue;
$this->onQueue(crons_queue());
}
/**
app/Jobs/ScheduledTaskJob.php
+1 -1
View File
@@ -65,7 +65,7 @@ class ScheduledTaskJob implements ShouldBeEncrypted, ShouldQueue
public function __construct($task)
{
$this->onQueue('high');
$this->onQueue(crons_queue());
$this->task = $task;
if ($service = $task->service()->first()) {
app/Livewire/Project/Application/Source.php
+9 -3
View File
@@ -3,6 +3,8 @@
namespace App\Livewire\Project\Application;
use App\Models\Application;
use App\Models\GithubApp;
use App\Models\GitlabApp;
use App\Models\PrivateKey;
use Illuminate\Foundation\Auth\Access\AuthorizesRequests;
use Livewire\Attributes\Locked;
@@ -21,7 +23,7 @@ class Source extends Component
#[Validate(['nullable', 'string'])]
public ?string $privateKeyName = null;
#[Validate(['nullable', 'integer'])]
#[Locked]
public ?int $privateKeyId = null;
#[Validate(['required', 'string'])]
@@ -103,7 +105,8 @@ class Source extends Component
{
try {
$this->authorize('update', $this->application);
$this->privateKeyId = $privateKeyId;
$key = PrivateKey::ownedByCurrentTeam()->findOrFail($privateKeyId);
$this->privateKeyId = $key->id;
$this->syncData(true);
$this->getPrivateKeys();
$this->application->refresh();
@@ -136,8 +139,11 @@ class Source extends Component
try {
$this->authorize('update', $this->application);
$allowedSourceTypes = [GithubApp::class, GitlabApp::class];
abort_unless(in_array($sourceType, $allowedSourceTypes, true), 404);
$source = $sourceType::ownedByCurrentTeam()->findOrFail($sourceId);
$this->application->update([
'source_id' => $sourceId,
'source_id' => $source->id,
'source_type' => $sourceType,
]);
app/Livewire/Project/Database/Import.php
+10 -4
View File
@@ -301,10 +301,16 @@ EOD;
} elseif ($stackServiceUuid) {
// ServiceDatabase route - look up the service database
$serviceUuid = data_get($this->parameters, 'service_uuid');
$service = Service::whereUuid($serviceUuid)->first();
if (! $service) {
abort(404);
}
$project = currentTeam()
->projects()
->select('id', 'uuid', 'team_id')
->where('uuid', data_get($this->parameters, 'project_uuid'))
->firstOrFail();
$environment = $project->environments()
->select('id', 'uuid', 'name', 'project_id')
->where('uuid', data_get($this->parameters, 'environment_uuid'))
->firstOrFail();
$service = $environment->services()->whereUuid($serviceUuid)->firstOrFail();
$resource = $service->databases()->whereUuid($stackServiceUuid)->first();
if (is_null($resource)) {
abort(404);
app/Livewire/Project/DeleteEnvironment.php
+5 -7
View File
@@ -4,12 +4,14 @@ namespace App\Livewire\Project;
use App\Models\Environment;
use Illuminate\Foundation\Auth\Access\AuthorizesRequests;
use Livewire\Attributes\Locked;
use Livewire\Component;
class DeleteEnvironment extends Component
{
use AuthorizesRequests;
#[Locked]
public int $environment_id;
public bool $disabled = false;
@@ -20,12 +22,8 @@ class DeleteEnvironment extends Component
public function mount()
{
try {
$this->environmentName = Environment::findOrFail($this->environment_id)->name;
$this->parameters = get_route_parameters();
} catch (\Exception $e) {
return handleError($e, $this);
}
$this->parameters = get_route_parameters();
$this->environmentName = Environment::ownedByCurrentTeam()->findOrFail($this->environment_id)->name;
}
public function delete()
@@ -33,7 +31,7 @@ class DeleteEnvironment extends Component
$this->validate([
'environment_id' => 'required|int',
]);
$environment = Environment::findOrFail($this->environment_id);
$environment = Environment::ownedByCurrentTeam()->findOrFail($this->environment_id);
$this->authorize('delete', $environment);
if ($environment->isEmpty()) {
app/Livewire/Project/New/GithubPrivateRepository.php
+17 -9
View File
@@ -9,6 +9,7 @@ use App\Rules\ValidGitBranch;
use App\Support\ValidationPatterns;
use Illuminate\Support\Facades\Http;
use Illuminate\Support\Facades\Route;
use Livewire\Attributes\Locked;
use Livewire\Component;
class GithubPrivateRepository extends Component
@@ -29,6 +30,7 @@ class GithubPrivateRepository extends Component
public int $selected_repository_id;
#[Locked]
public int $selected_github_app_id;
public string $selected_repository_owner;
@@ -37,8 +39,6 @@ class GithubPrivateRepository extends Component
public string $selected_branch_name = 'main';
public string $token;
public $repositories;
public int $total_repositories_count = 0;
@@ -71,7 +71,10 @@ class GithubPrivateRepository extends Component
$this->parameters = get_route_parameters();
$this->query = request()->query();
$this->repositories = $this->branches = collect();
$this->github_apps = GithubApp::private();
$this->github_apps = GithubApp::where('team_id', currentTeam()->id)
->where('is_public', false)
->whereNotNull('app_id')
->get();
}
public function updatedSelectedRepositoryId(): void
@@ -96,22 +99,25 @@ class GithubPrivateRepository extends Component
}
}
public function loadRepositories($github_app_id)
public function loadRepositories(int $github_app_id): void
{
$this->repositories = collect();
$this->branches = collect();
$this->total_branches_count = 0;
$this->page = 1;
$this->selected_github_app_id = $github_app_id;
$this->github_app = GithubApp::where('id', $github_app_id)->first();
$this->token = generateGithubInstallationToken($this->github_app);
$repositories = loadRepositoryByPage($this->github_app, $this->token, $this->page);
$this->github_app = GithubApp::where('team_id', currentTeam()->id)
->where('is_public', false)
->whereNotNull('app_id')
->findOrFail($github_app_id);
$token = generateGithubInstallationToken($this->github_app);
$repositories = loadRepositoryByPage($this->github_app, $token, $this->page);
$this->total_repositories_count = $repositories['total_count'];
$this->repositories = $this->repositories->concat(collect($repositories['repositories']));
if ($this->repositories->count() < $this->total_repositories_count) {
while ($this->repositories->count() < $this->total_repositories_count) {
$this->page++;
$repositories = loadRepositoryByPage($this->github_app, $this->token, $this->page);
$repositories = loadRepositoryByPage($this->github_app, $token, $this->page);
$this->total_repositories_count = $repositories['total_count'];
$this->repositories = $this->repositories->concat(collect($repositories['repositories']));
}
@@ -142,7 +148,9 @@ class GithubPrivateRepository extends Component
protected function loadBranchByPage()
{
$response = Http::GitHub($this->github_app->api_url, $this->token)
$token = generateGithubInstallationToken($this->github_app);
$response = Http::GitHub($this->github_app->api_url, $token)
->timeout(20)
->retry(3, 200, throw: false)
->get("/repos/{$this->selected_repository_owner}/{$this->selected_repository_repo}/branches", [
app/Livewire/Project/Service/DatabaseBackups.php
+10 -4
View File
@@ -28,10 +28,16 @@ class DatabaseBackups extends Component
try {
$this->parameters = get_route_parameters();
$this->query = request()->query();
$this->service = Service::whereUuid($this->parameters['service_uuid'])->first();
if (! $this->service) {
return redirect()->route('dashboard');
}
$project = currentTeam()
->projects()
->select('id', 'uuid', 'team_id')
->where('uuid', $this->parameters['project_uuid'])
->firstOrFail();
$environment = $project->environments()
->select('id', 'uuid', 'name', 'project_id')
->where('uuid', $this->parameters['environment_uuid'])
->firstOrFail();
$this->service = $environment->services()->whereUuid($this->parameters['service_uuid'])->firstOrFail();
$this->authorize('view', $this->service);
$this->serviceDatabase = $this->service->databases()->whereUuid($this->parameters['stack_service_uuid'])->first();
app/Livewire/Project/Service/Heading.php
+30
View File
@@ -7,12 +7,15 @@ use App\Actions\Service\StartService;
use App\Actions\Service\StopService;
use App\Enums\ProcessStatus;
use App\Models\Service;
use Illuminate\Foundation\Auth\Access\AuthorizesRequests;
use Illuminate\Support\Facades\Auth;
use Livewire\Component;
use Spatie\Activitylog\Models\Activity;
class Heading extends Component
{
use AuthorizesRequests;
public Service $service;
public array $parameters;
@@ -27,6 +30,8 @@ class Heading extends Component
public function mount()
{
$this->authorizeService('view');
if (str($this->service->status)->contains('running') && is_null($this->service->config_hash)) {
$this->service->isConfigurationChanged(true);
$this->dispatch('configurationChanged');
@@ -47,6 +52,8 @@ class Heading extends Component
public function checkStatus()
{
$this->authorizeService('view');
if ($this->service->server->isFunctional()) {
GetContainersStatus::dispatch($this->service->server);
} else {
@@ -61,6 +68,8 @@ class Heading extends Component
public function serviceChecked()
{
$this->authorizeService('view');
try {
$this->service->applications->each(function ($application) {
$application->refresh();
@@ -82,6 +91,8 @@ class Heading extends Component
public function checkDeployments()
{
$this->authorizeService('view');
try {
$activity = Activity::where('properties->type_uuid', $this->service->uuid)->latest()->first();
$status = data_get($activity, 'properties.status');
@@ -99,12 +110,16 @@ class Heading extends Component
public function start()
{
$this->authorizeService('deploy');
$activity = StartService::run($this->service, pullLatestImages: true);
$this->dispatch('activityMonitor', $activity->id);
}
public function forceDeploy()
{
$this->authorizeService('deploy');
try {
$activities = Activity::where('properties->type_uuid', $this->service->uuid)
->where(function ($q) {
@@ -124,6 +139,8 @@ class Heading extends Component
public function stop()
{
$this->authorizeService('stop');
try {
StopService::dispatch($this->service, false, $this->docker_cleanup);
} catch (\Exception $e) {
@@ -133,6 +150,8 @@ class Heading extends Component
public function restart()
{
$this->authorizeService('deploy');
$this->checkDeployments();
if ($this->isDeploymentProgress) {
$this->dispatch('error', 'There is a deployment in progress.');
@@ -145,6 +164,8 @@ class Heading extends Component
public function pullAndRestartEvent()
{
$this->authorizeService('deploy');
$this->checkDeployments();
if ($this->isDeploymentProgress) {
$this->dispatch('error', 'There is a deployment in progress.');
@@ -155,6 +176,15 @@ class Heading extends Component
$this->dispatch('activityMonitor', $activity->id);
}
private function authorizeService(string $ability): void
{
$this->service = Service::ownedByCurrentTeam()
->whereKey($this->service->getKey())
->firstOrFail();
$this->authorize($ability, $this->service);
}
public function render()
{
return view('livewire.project.service.heading', [
app/Livewire/Project/Service/Index.php
+10 -4
View File
@@ -108,10 +108,16 @@ class Index extends Component
$this->parameters = get_route_parameters();
$this->query = request()->query();
$this->currentRoute = request()->route()->getName();
$this->service = Service::whereUuid($this->parameters['service_uuid'])->first();
if (! $this->service) {
return redirect()->route('dashboard');
}
$project = currentTeam()
->projects()
->select('id', 'uuid', 'team_id')
->where('uuid', $this->parameters['project_uuid'])
->firstOrFail();
$environment = $project->environments()
->select('id', 'uuid', 'name', 'project_id')
->where('uuid', $this->parameters['environment_uuid'])
->firstOrFail();
$this->service = $environment->services()->whereUuid($this->parameters['service_uuid'])->firstOrFail();
$this->authorize('view', $this->service);
$service = $this->service->applications()->whereUuid($this->parameters['stack_service_uuid'])->first();
if ($service) {
app/Livewire/Project/Shared/Destination.php
+27 -11
View File
@@ -110,15 +110,23 @@ class Destination extends Component
public function promote(int $network_id, int $server_id)
{
$main_destination = $this->resource->destination;
$this->resource->update([
'destination_id' => $network_id,
'destination_type' => StandaloneDocker::class,
]);
$this->resource->additional_networks()->detach($network_id, ['server_id' => $server_id]);
$this->resource->additional_networks()->attach($main_destination->id, ['server_id' => $main_destination->server->id]);
$this->refreshServers();
$this->resource->refresh();
try {
$server = Server::ownedByCurrentTeam()->findOrFail($server_id);
$network = StandaloneDocker::ownedByCurrentTeam()->findOrFail($network_id);
$this->authorize('update', $this->resource);
$main_destination = $this->resource->destination;
$this->resource->update([
'destination_id' => $network->id,
'destination_type' => StandaloneDocker::class,
]);
$this->resource->additional_networks()->detach($network->id, ['server_id' => $server->id]);
$this->resource->additional_networks()->attach($main_destination->id, ['server_id' => $main_destination->server->id]);
$this->refreshServers();
$this->resource->refresh();
} catch (\Exception $e) {
return handleError($e, $this);
}
}
public function refreshServers()
@@ -130,8 +138,16 @@ class Destination extends Component
public function addServer(int $network_id, int $server_id)
{
$this->resource->additional_networks()->attach($network_id, ['server_id' => $server_id]);
$this->dispatch('refresh');
try {
$server = Server::ownedByCurrentTeam()->findOrFail($server_id);
$network = StandaloneDocker::ownedByCurrentTeam()->findOrFail($network_id);
$this->authorize('update', $this->resource);
$this->resource->additional_networks()->attach($network->id, ['server_id' => $server->id]);
$this->dispatch('refresh');
} catch (\Exception $e) {
return handleError($e, $this);
}
}
public function removeServer(int $network_id, int $server_id, $password, $selectedActions = [])
app/Livewire/Security/ApiTokens.php
+8 -5
View File
@@ -5,6 +5,7 @@ namespace App\Livewire\Security;
use App\Models\InstanceSettings;
use Illuminate\Foundation\Auth\Access\AuthorizesRequests;
use Laravel\Sanctum\PersonalAccessToken;
use Livewire\Attributes\Locked;
use Livewire\Component;
class ApiTokens extends Component
@@ -29,8 +30,10 @@ class ApiTokens extends Component
public $isApiEnabled;
#[Locked]
public bool $canUseRootPermissions = false;
#[Locked]
public bool $canUseWritePermissions = false;
public function render()
@@ -54,7 +57,7 @@ class ApiTokens extends Component
public function updatedPermissions($permissionToUpdate)
{
// Check if user is trying to use restricted permissions
if ($permissionToUpdate == 'root' && ! $this->canUseRootPermissions) {
if ($permissionToUpdate == 'root' && ! auth()->user()->can('useRootPermissions', PersonalAccessToken::class)) {
$this->dispatch('error', 'You do not have permission to use root permissions.');
// Remove root from permissions if it was somehow added
$this->permissions = array_diff($this->permissions, ['root']);
@@ -62,7 +65,7 @@ class ApiTokens extends Component
return;
}
if (in_array($permissionToUpdate, ['write', 'write:sensitive']) && ! $this->canUseWritePermissions) {
if (in_array($permissionToUpdate, ['write', 'write:sensitive'], true) && ! auth()->user()->can('useWritePermissions', PersonalAccessToken::class)) {
$this->dispatch('error', 'You do not have permission to use write permissions.');
// Remove write permissions if they were somehow added
$this->permissions = array_diff($this->permissions, ['write', 'write:sensitive']);
@@ -72,7 +75,7 @@ class ApiTokens extends Component
if ($permissionToUpdate == 'root') {
$this->permissions = ['root'];
} elseif ($permissionToUpdate == 'read:sensitive' && ! in_array('read', $this->permissions)) {
} elseif ($permissionToUpdate == 'read:sensitive' && ! in_array('read', $this->permissions, true)) {
$this->permissions[] = 'read';
} elseif ($permissionToUpdate == 'deploy') {
$this->permissions = ['deploy'];
@@ -90,11 +93,11 @@ class ApiTokens extends Component
$this->authorize('create', PersonalAccessToken::class);
// Validate permissions based on user role
if (in_array('root', $this->permissions) && ! $this->canUseRootPermissions) {
if (in_array('root', $this->permissions, true) && ! auth()->user()->can('useRootPermissions', PersonalAccessToken::class)) {
throw new \Exception('You do not have permission to create tokens with root permissions.');
}
if (array_intersect(['write', 'write:sensitive'], $this->permissions) && ! $this->canUseWritePermissions) {
if (array_intersect(['write', 'write:sensitive'], $this->permissions) && ! auth()->user()->can('useWritePermissions', PersonalAccessToken::class)) {
throw new \Exception('You do not have permission to create tokens with write permissions.');
}
app/Livewire/Source/Github/Change.php
+23
View File
@@ -7,7 +7,9 @@ use App\Models\GithubApp;
use App\Models\PrivateKey;
use App\Rules\SafeExternalUrl;
use Illuminate\Foundation\Auth\Access\AuthorizesRequests;
use Illuminate\Support\Facades\Cache;
use Illuminate\Support\Facades\Http;
use Illuminate\Support\Str;
use Lcobucci\JWT\Configuration;
use Lcobucci\JWT\Signer\Key\InMemory;
use Lcobucci\JWT\Signer\Rsa\Sha256;
@@ -72,6 +74,8 @@ class Change extends Component
public $privateKeys;
public string $manifestState = '';
protected function rules(): array
{
return [
@@ -147,6 +151,24 @@ class Change extends Component
}
}
private function githubAppSetupStateCacheKey(string $state): string
{
return 'github-app-setup-state:'.hash('sha256', $state);
}
private function createGithubAppSetupState(string $action): string
{
$state = Str::random(64);
Cache::put($this->githubAppSetupStateCacheKey($state), [
'action' => $action,
'github_app_id' => $this->github_app->id,
'team_id' => $this->github_app->team_id,
], now()->addMinutes(60));
return $state;
}
public function checkPermissions()
{
try {
@@ -211,6 +233,7 @@ class Change extends Component
// Override name with kebab case for display
$this->name = str($this->github_app->name)->kebab();
$this->fqdn = $settings->fqdn;
$this->manifestState = $this->createGithubAppSetupState('manifest');
if ($settings->public_ipv4) {
$this->ipv4 = 'http://'.$settings->public_ipv4.':'.config('app.port');
app/Models/GithubApp.php
-20
View File
@@ -73,26 +73,6 @@ class GithubApp extends BaseModel
});
}
public static function public()
{
return GithubApp::where(function ($query) {
$query->where(function ($q) {
$q->where('team_id', currentTeam()->id)
->orWhere('is_system_wide', true);
})->where('is_public', true);
})->whereNotNull('app_id')->get();
}
public static function private()
{
return GithubApp::where(function ($query) {
$query->where(function ($q) {
$q->where('team_id', currentTeam()->id)
->orWhere('is_system_wide', true);
})->where('is_public', false);
})->whereNotNull('app_id')->get();
}
public function team()
{
return $this->belongsTo(Team::class);