fix(backup): use shared compression for database dumps

Centralize compressor selection and apply helper-image compression to dump-all database backups.
This commit is contained in:
Andras Bacsai committed 2026-08-15 15:51:11 +02:00
1 parent 909e776fbc
commit 0a6db15216
5 files changed
+106 -15

No files matched your search

+16 -3
View File
@@ -18,6 +18,7 @@ use App\Notifications\Database\BackupFailed;
use App\Notifications\Database\BackupSuccess;
use App\Notifications\Database\BackupSuccessWithS3Warning;
use App\Rules\SafeWebhookUrl;
use App\Support\BackupCompression;
use App\Support\ClickhouseBackupCommand;
use Carbon\Carbon;
use Illuminate\Bus\Queueable;
@@ -609,7 +610,8 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue
}
$escapedUsername = escapeshellarg($this->database->postgres_user);
if ($this->backup->dump_all) {
$backupCommand .= " $this->container_name pg_dumpall --username $escapedUsername | gzip > $this->backup_location";
$backupCommand .= " $this->container_name pg_dumpall --username $escapedUsername";
$backupCommand = $this->buildCompressedDumpCommand($backupCommand).' > '.escapeshellarg($this->backup_location);
} else {
// Validate and escape database name to prevent command injection
validateShellSafePath($database, 'database name');
@@ -635,7 +637,8 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue
$commands[] = 'mkdir -p '.$this->backup_dir;
$escapedPassword = escapeshellarg($this->database->mysql_root_password);
if ($this->backup->dump_all) {
$commands[] = "docker exec $this->container_name mysqldump -u root -p$escapedPassword --all-databases --single-transaction --quick --lock-tables=false --compress | gzip > $this->backup_location";
$dumpCommand = "docker exec $this->container_name mysqldump -u root -p$escapedPassword --all-databases --single-transaction --quick --lock-tables=false";
$commands[] = $this->buildCompressedDumpCommand($dumpCommand).' > '.escapeshellarg($this->backup_location);
} else {
// Validate and escape database name to prevent command injection
validateShellSafePath($database, 'database name');
@@ -659,7 +662,8 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue
$commands[] = 'mkdir -p '.$this->backup_dir;
$escapedPassword = escapeshellarg($this->database->mariadb_root_password);
if ($this->backup->dump_all) {
$commands[] = "docker exec $this->container_name mariadb-dump -u root -p$escapedPassword --all-databases --single-transaction --quick --lock-tables=false --compress > $this->backup_location";
$dumpCommand = "docker exec $this->container_name mariadb-dump -u root -p$escapedPassword --all-databases --single-transaction --quick --lock-tables=false";
$commands[] = $this->buildCompressedDumpCommand($dumpCommand).' > '.escapeshellarg($this->backup_location);
} else {
// Validate and escape database name to prevent command injection
validateShellSafePath($database, 'database name');
@@ -806,6 +810,15 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue
return "{$helperImage}:{$latestVersion}";
}
private function buildCompressedDumpCommand(string $dumpCommand): string
{
$cpuPercentage = BackupCompression::cpuPercentage($this->server->settings->backup_compression_cpu_percentage);
$compressorCommand = BackupCompression::compressorCommand($cpuPercentage);
$script = "compressor=\$({$compressorCommand}); exec \$compressor";
return $dumpCommand.' | docker run --rm -i '.escapeshellarg($this->getFullImageName()).' sh -c '.escapeshellarg($script);
}
private function markStaleExecutionsAsFailed(): void
{
try {
+5 -12
View File
@@ -8,6 +8,7 @@ use App\Models\ScheduledVolumeBackup;
use App\Models\ScheduledVolumeBackupExecution;
use App\Models\Server;
use App\Rules\SafeWebhookUrl;
use App\Support\BackupCompression;
use Carbon\Carbon;
use Illuminate\Bus\Queueable;
use Illuminate\Contracts\Queue\ShouldBeEncrypted;
@@ -77,15 +78,14 @@ class VolumeBackupJob implements ShouldBeEncrypted, ShouldQueue
$source = $this->backup->sourcePath();
$containerName = 'volume-backup-'.$this->execution->uuid;
$image = coolifyHelperImage().':'.getHelperVersion();
$compressionCpuPercentage = $this->compressionCpuPercentage($server);
$compressionCpuPercentage = BackupCompression::cpuPercentage($server->settings->backup_compression_cpu_percentage);
$this->logCompressorInDevelopment($image, $server, $compressionCpuPercentage);
$verifySourceCommand = $target instanceof LocalPersistentVolume && blank($target->host_path)
? 'docker volume inspect '.escapeshellarg($source).' >/dev/null'
: 'test -d '.escapeshellarg($source);
$archiveScript = "compressor='gzip -3'; "
."if command -v pigz >/dev/null 2>&1; then compressor=\"pigz -3 -p \$(( (\$(nproc) * {$compressionCpuPercentage} + 99) / 100 ))\"; fi; "
.'tar -I "$compressor" -cf - -C /volume .';
$compressorCommand = BackupCompression::compressorCommand($compressionCpuPercentage);
$archiveScript = "compressor=\$({$compressorCommand}); tar -I \"\$compressor\" -cf - -C /volume .";
$archiveCommand = 'docker run --rm --name '.escapeshellarg($containerName)
.' -v '.escapeshellarg($source.':/volume:ro')
.' '.escapeshellarg($image)
@@ -344,7 +344,7 @@ class VolumeBackupJob implements ShouldBeEncrypted, ShouldQueue
return;
}
$script = "if command -v pigz >/dev/null 2>&1; then printf 'pigz -3 -p %s' \"\$(( (\$(nproc) * {$compressionCpuPercentage} + 99) / 100 ))\"; else printf 'gzip -3'; fi";
$script = BackupCompression::compressorCommand($compressionCpuPercentage);
$compressor = instant_remote_process(
['docker run --rm '.escapeshellarg($image).' sh -c '.escapeshellarg($script)],
$server,
@@ -361,13 +361,6 @@ class VolumeBackupJob implements ShouldBeEncrypted, ShouldQueue
]);
}
private function compressionCpuPercentage(Server $server): int
{
$percentage = (int) ($server->settings->backup_compression_cpu_percentage ?? 25);
return in_array($percentage, [25, 50, 75, 100], true) ? $percentage : 25;
}
private function removeExpiredBackups(Server $server): void
{
if ($this->hasRetentionLimits(
+20
View File
@@ -0,0 +1,20 @@
<?php
namespace App\Support;
final class BackupCompression
{
public static function cpuPercentage(int|string|null $configuredPercentage): int
{
$percentage = (int) $configuredPercentage;
return in_array($percentage, [25, 50, 75, 100], true) ? $percentage : 25;
}
public static function compressorCommand(int $cpuPercentage): string
{
$cpuPercentage = self::cpuPercentage($cpuPercentage);
return "if command -v pigz >/dev/null 2>&1; then printf 'pigz -3 -p %s' \"\$(( (\$(nproc) * {$cpuPercentage} + 99) / 100 ))\"; else printf 'gzip -3'; fi";
}
}
@@ -1,9 +1,12 @@
<?php
use App\Jobs\DatabaseBackupJob;
use App\Models\InstanceSettings;
use App\Models\S3Storage;
use App\Models\ScheduledDatabaseBackup;
use App\Models\ScheduledDatabaseBackupExecution;
use App\Models\Server;
use App\Models\ServerSetting;
use App\Models\Team;
use Illuminate\Foundation\Testing\RefreshDatabase;
use Illuminate\Support\Facades\Schema;
@@ -322,3 +325,40 @@ test('database backup job escapes the S3 copy destination argument', function ()
->and($source)->toContain('mc cp {$escapedBackupLocation} {$escapedS3Destination}')
->and($source)->not->toContain('mc cp $this->backup_location temporary/$bucket{$this->backup_dir}/');
});
test('database dump compression uses the helper image and shared CPU setting', function (int $compressionCpuPercentage) {
InstanceSettings::unguarded(fn () => InstanceSettings::create(['id' => 0]));
$backup = new ScheduledDatabaseBackup(['timeout' => 3600]);
$job = new DatabaseBackupJob($backup);
$server = new Server;
$server->setRelation('settings', new ServerSetting([
'backup_compression_cpu_percentage' => $compressionCpuPercentage,
]));
$job->server = $server;
$command = (new ReflectionClass($job))
->getMethod('buildCompressedDumpCommand')
->invoke($job, 'docker exec database pg_dumpall');
expect($command)
->toStartWith('docker exec database pg_dumpall | docker run --rm -i')
->toContain('coolify-helper')
->toContain('command -v pigz')
->toContain('pigz -3 -p')
->toContain("\$(nproc) * {$compressionCpuPercentage} + 99")
->toContain('gzip -3');
})->with([
'low' => 25,
'high' => 75,
]);
test('all dump all database commands use shared helper compression', function () {
$source = file_get_contents(app_path('Jobs/DatabaseBackupJob.php'));
expect($source)
->toContain('$this->buildCompressedDumpCommand($backupCommand)')
->toContain('mysqldump -u root')
->toContain('mariadb-dump -u root')
->and(substr_count($source, '$this->buildCompressedDumpCommand($dumpCommand)'))->toBe(2)
->and($source)->not->toContain('| gzip >');
});
+25
View File
@@ -0,0 +1,25 @@
<?php
use App\Support\BackupCompression;
it('normalizes supported backup compression CPU percentages', function (mixed $configuredPercentage, int $expected) {
expect(BackupCompression::cpuPercentage($configuredPercentage))->toBe($expected);
})->with([
'null defaults to low' => [null, 25],
'unsupported value defaults to low' => [30, 25],
'numeric string is accepted' => ['50', 50],
'low' => [25, 25],
'medium' => [50, 50],
'high' => [75, 75],
'maximum' => [100, 100],
]);
it('builds the shared pigz command with gzip fallback', function () {
$command = BackupCompression::compressorCommand(75);
expect($command)
->toContain('command -v pigz')
->toContain('pigz -3 -p')
->toContain('$(nproc) * 75 + 99')
->toContain('gzip -3');
});