mirror of
https://github.com/tiennm99/coolify.git
synced 2026-10-13 04:14:59 +00:00
fix(backup): use shared compression for database dumps
Centralize compressor selection and apply helper-image compression to dump-all database backups.
This commit is contained in:
1 parent
909e776fbc
commit
0a6db15216
5 files changed
+106
-15
No files matched your search
@@ -18,6 +18,7 @@ use App\Notifications\Database\BackupFailed;
|
||||
use App\Notifications\Database\BackupSuccess;
|
||||
use App\Notifications\Database\BackupSuccessWithS3Warning;
|
||||
use App\Rules\SafeWebhookUrl;
|
||||
use App\Support\BackupCompression;
|
||||
use App\Support\ClickhouseBackupCommand;
|
||||
use Carbon\Carbon;
|
||||
use Illuminate\Bus\Queueable;
|
||||
@@ -609,7 +610,8 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue
|
||||
}
|
||||
$escapedUsername = escapeshellarg($this->database->postgres_user);
|
||||
if ($this->backup->dump_all) {
|
||||
$backupCommand .= " $this->container_name pg_dumpall --username $escapedUsername | gzip > $this->backup_location";
|
||||
$backupCommand .= " $this->container_name pg_dumpall --username $escapedUsername";
|
||||
$backupCommand = $this->buildCompressedDumpCommand($backupCommand).' > '.escapeshellarg($this->backup_location);
|
||||
} else {
|
||||
// Validate and escape database name to prevent command injection
|
||||
validateShellSafePath($database, 'database name');
|
||||
@@ -635,7 +637,8 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue
|
||||
$commands[] = 'mkdir -p '.$this->backup_dir;
|
||||
$escapedPassword = escapeshellarg($this->database->mysql_root_password);
|
||||
if ($this->backup->dump_all) {
|
||||
$commands[] = "docker exec $this->container_name mysqldump -u root -p$escapedPassword --all-databases --single-transaction --quick --lock-tables=false --compress | gzip > $this->backup_location";
|
||||
$dumpCommand = "docker exec $this->container_name mysqldump -u root -p$escapedPassword --all-databases --single-transaction --quick --lock-tables=false";
|
||||
$commands[] = $this->buildCompressedDumpCommand($dumpCommand).' > '.escapeshellarg($this->backup_location);
|
||||
} else {
|
||||
// Validate and escape database name to prevent command injection
|
||||
validateShellSafePath($database, 'database name');
|
||||
@@ -659,7 +662,8 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue
|
||||
$commands[] = 'mkdir -p '.$this->backup_dir;
|
||||
$escapedPassword = escapeshellarg($this->database->mariadb_root_password);
|
||||
if ($this->backup->dump_all) {
|
||||
$commands[] = "docker exec $this->container_name mariadb-dump -u root -p$escapedPassword --all-databases --single-transaction --quick --lock-tables=false --compress > $this->backup_location";
|
||||
$dumpCommand = "docker exec $this->container_name mariadb-dump -u root -p$escapedPassword --all-databases --single-transaction --quick --lock-tables=false";
|
||||
$commands[] = $this->buildCompressedDumpCommand($dumpCommand).' > '.escapeshellarg($this->backup_location);
|
||||
} else {
|
||||
// Validate and escape database name to prevent command injection
|
||||
validateShellSafePath($database, 'database name');
|
||||
@@ -806,6 +810,15 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue
|
||||
return "{$helperImage}:{$latestVersion}";
|
||||
}
|
||||
|
||||
private function buildCompressedDumpCommand(string $dumpCommand): string
|
||||
{
|
||||
$cpuPercentage = BackupCompression::cpuPercentage($this->server->settings->backup_compression_cpu_percentage);
|
||||
$compressorCommand = BackupCompression::compressorCommand($cpuPercentage);
|
||||
$script = "compressor=\$({$compressorCommand}); exec \$compressor";
|
||||
|
||||
return $dumpCommand.' | docker run --rm -i '.escapeshellarg($this->getFullImageName()).' sh -c '.escapeshellarg($script);
|
||||
}
|
||||
|
||||
private function markStaleExecutionsAsFailed(): void
|
||||
{
|
||||
try {
|
||||
|
||||
@@ -8,6 +8,7 @@ use App\Models\ScheduledVolumeBackup;
|
||||
use App\Models\ScheduledVolumeBackupExecution;
|
||||
use App\Models\Server;
|
||||
use App\Rules\SafeWebhookUrl;
|
||||
use App\Support\BackupCompression;
|
||||
use Carbon\Carbon;
|
||||
use Illuminate\Bus\Queueable;
|
||||
use Illuminate\Contracts\Queue\ShouldBeEncrypted;
|
||||
@@ -77,15 +78,14 @@ class VolumeBackupJob implements ShouldBeEncrypted, ShouldQueue
|
||||
$source = $this->backup->sourcePath();
|
||||
$containerName = 'volume-backup-'.$this->execution->uuid;
|
||||
$image = coolifyHelperImage().':'.getHelperVersion();
|
||||
$compressionCpuPercentage = $this->compressionCpuPercentage($server);
|
||||
$compressionCpuPercentage = BackupCompression::cpuPercentage($server->settings->backup_compression_cpu_percentage);
|
||||
$this->logCompressorInDevelopment($image, $server, $compressionCpuPercentage);
|
||||
$verifySourceCommand = $target instanceof LocalPersistentVolume && blank($target->host_path)
|
||||
? 'docker volume inspect '.escapeshellarg($source).' >/dev/null'
|
||||
: 'test -d '.escapeshellarg($source);
|
||||
|
||||
$archiveScript = "compressor='gzip -3'; "
|
||||
."if command -v pigz >/dev/null 2>&1; then compressor=\"pigz -3 -p \$(( (\$(nproc) * {$compressionCpuPercentage} + 99) / 100 ))\"; fi; "
|
||||
.'tar -I "$compressor" -cf - -C /volume .';
|
||||
$compressorCommand = BackupCompression::compressorCommand($compressionCpuPercentage);
|
||||
$archiveScript = "compressor=\$({$compressorCommand}); tar -I \"\$compressor\" -cf - -C /volume .";
|
||||
$archiveCommand = 'docker run --rm --name '.escapeshellarg($containerName)
|
||||
.' -v '.escapeshellarg($source.':/volume:ro')
|
||||
.' '.escapeshellarg($image)
|
||||
@@ -344,7 +344,7 @@ class VolumeBackupJob implements ShouldBeEncrypted, ShouldQueue
|
||||
return;
|
||||
}
|
||||
|
||||
$script = "if command -v pigz >/dev/null 2>&1; then printf 'pigz -3 -p %s' \"\$(( (\$(nproc) * {$compressionCpuPercentage} + 99) / 100 ))\"; else printf 'gzip -3'; fi";
|
||||
$script = BackupCompression::compressorCommand($compressionCpuPercentage);
|
||||
$compressor = instant_remote_process(
|
||||
['docker run --rm '.escapeshellarg($image).' sh -c '.escapeshellarg($script)],
|
||||
$server,
|
||||
@@ -361,13 +361,6 @@ class VolumeBackupJob implements ShouldBeEncrypted, ShouldQueue
|
||||
]);
|
||||
}
|
||||
|
||||
private function compressionCpuPercentage(Server $server): int
|
||||
{
|
||||
$percentage = (int) ($server->settings->backup_compression_cpu_percentage ?? 25);
|
||||
|
||||
return in_array($percentage, [25, 50, 75, 100], true) ? $percentage : 25;
|
||||
}
|
||||
|
||||
private function removeExpiredBackups(Server $server): void
|
||||
{
|
||||
if ($this->hasRetentionLimits(
|
||||
|
||||
@@ -0,0 +1,20 @@
|
||||
<?php
|
||||
|
||||
namespace App\Support;
|
||||
|
||||
final class BackupCompression
|
||||
{
|
||||
public static function cpuPercentage(int|string|null $configuredPercentage): int
|
||||
{
|
||||
$percentage = (int) $configuredPercentage;
|
||||
|
||||
return in_array($percentage, [25, 50, 75, 100], true) ? $percentage : 25;
|
||||
}
|
||||
|
||||
public static function compressorCommand(int $cpuPercentage): string
|
||||
{
|
||||
$cpuPercentage = self::cpuPercentage($cpuPercentage);
|
||||
|
||||
return "if command -v pigz >/dev/null 2>&1; then printf 'pigz -3 -p %s' \"\$(( (\$(nproc) * {$cpuPercentage} + 99) / 100 ))\"; else printf 'gzip -3'; fi";
|
||||
}
|
||||
}
|
||||
@@ -1,9 +1,12 @@
|
||||
<?php
|
||||
|
||||
use App\Jobs\DatabaseBackupJob;
|
||||
use App\Models\InstanceSettings;
|
||||
use App\Models\S3Storage;
|
||||
use App\Models\ScheduledDatabaseBackup;
|
||||
use App\Models\ScheduledDatabaseBackupExecution;
|
||||
use App\Models\Server;
|
||||
use App\Models\ServerSetting;
|
||||
use App\Models\Team;
|
||||
use Illuminate\Foundation\Testing\RefreshDatabase;
|
||||
use Illuminate\Support\Facades\Schema;
|
||||
@@ -322,3 +325,40 @@ test('database backup job escapes the S3 copy destination argument', function ()
|
||||
->and($source)->toContain('mc cp {$escapedBackupLocation} {$escapedS3Destination}')
|
||||
->and($source)->not->toContain('mc cp $this->backup_location temporary/$bucket{$this->backup_dir}/');
|
||||
});
|
||||
|
||||
test('database dump compression uses the helper image and shared CPU setting', function (int $compressionCpuPercentage) {
|
||||
InstanceSettings::unguarded(fn () => InstanceSettings::create(['id' => 0]));
|
||||
$backup = new ScheduledDatabaseBackup(['timeout' => 3600]);
|
||||
$job = new DatabaseBackupJob($backup);
|
||||
$server = new Server;
|
||||
$server->setRelation('settings', new ServerSetting([
|
||||
'backup_compression_cpu_percentage' => $compressionCpuPercentage,
|
||||
]));
|
||||
$job->server = $server;
|
||||
|
||||
$command = (new ReflectionClass($job))
|
||||
->getMethod('buildCompressedDumpCommand')
|
||||
->invoke($job, 'docker exec database pg_dumpall');
|
||||
|
||||
expect($command)
|
||||
->toStartWith('docker exec database pg_dumpall | docker run --rm -i')
|
||||
->toContain('coolify-helper')
|
||||
->toContain('command -v pigz')
|
||||
->toContain('pigz -3 -p')
|
||||
->toContain("\$(nproc) * {$compressionCpuPercentage} + 99")
|
||||
->toContain('gzip -3');
|
||||
})->with([
|
||||
'low' => 25,
|
||||
'high' => 75,
|
||||
]);
|
||||
|
||||
test('all dump all database commands use shared helper compression', function () {
|
||||
$source = file_get_contents(app_path('Jobs/DatabaseBackupJob.php'));
|
||||
|
||||
expect($source)
|
||||
->toContain('$this->buildCompressedDumpCommand($backupCommand)')
|
||||
->toContain('mysqldump -u root')
|
||||
->toContain('mariadb-dump -u root')
|
||||
->and(substr_count($source, '$this->buildCompressedDumpCommand($dumpCommand)'))->toBe(2)
|
||||
->and($source)->not->toContain('| gzip >');
|
||||
});
|
||||
@@ -0,0 +1,25 @@
|
||||
<?php
|
||||
|
||||
use App\Support\BackupCompression;
|
||||
|
||||
it('normalizes supported backup compression CPU percentages', function (mixed $configuredPercentage, int $expected) {
|
||||
expect(BackupCompression::cpuPercentage($configuredPercentage))->toBe($expected);
|
||||
})->with([
|
||||
'null defaults to low' => [null, 25],
|
||||
'unsupported value defaults to low' => [30, 25],
|
||||
'numeric string is accepted' => ['50', 50],
|
||||
'low' => [25, 25],
|
||||
'medium' => [50, 50],
|
||||
'high' => [75, 75],
|
||||
'maximum' => [100, 100],
|
||||
]);
|
||||
|
||||
it('builds the shared pigz command with gzip fallback', function () {
|
||||
$command = BackupCompression::compressorCommand(75);
|
||||
|
||||
expect($command)
|
||||
->toContain('command -v pigz')
|
||||
->toContain('pigz -3 -p')
|
||||
->toContain('$(nproc) * 75 + 99')
|
||||
->toContain('gzip -3');
|
||||
});
|
||||
Reference in new issue
Block a user