diff --git a/apps/dokploy/__test__/utils/backups.test.ts b/apps/dokploy/__test__/utils/backups.test.ts index 2c1e5decc9..fb821729db 100644 --- a/apps/dokploy/__test__/utils/backups.test.ts +++ b/apps/dokploy/__test__/utils/backups.test.ts @@ -1,4 +1,9 @@ -import { normalizeS3Path } from "@dokploy/server/utils/backups/utils"; +import type { BackupSchedule } from "@dokploy/server/services/backup"; +import { + getBackupCommand, + getPostgresBackupCommand, + normalizeS3Path, +} from "@dokploy/server/utils/backups/utils"; import { describe, expect, test } from "vitest"; describe("normalizeS3Path", () => { @@ -59,3 +64,38 @@ describe("normalizeS3Path", () => { expect(normalizeS3Path("instance-backups")).toBe("instance-backups/"); }); }); + +describe("getBackupCommand", () => { + const backup = { + backupType: "database", + databaseType: "postgres", + database: "mydb", + postgres: { appName: "my-app", databaseUser: "postgres" }, + } as unknown as BackupSchedule; + const rcloneCommand = + 'rclone rcat --s3-region=auto ":s3:bucket/my-app/backup.sql.gz"'; + const buildScript = () => + getBackupCommand(backup, rcloneCommand, "/tmp/backup.log"); + + test("should run the database dump exactly once", () => { + const script = buildScript(); + const dumpCommand = getPostgresBackupCommand("mydb", "postgres"); + expect(script.split(dumpCommand).length - 1).toBe(1); + }); + + test("should stream the dump directly into rclone", () => { + expect(buildScript()).toContain(`| ${rcloneCommand}`); + }); + + test("should keep dump and upload failures distinguishable", () => { + const script = buildScript(); + expect(script).toContain("Error: Backup failed"); + expect(script).toContain("Error: Upload to S3 failed"); + }); + + test("should clean up the partial object when a stream fails", () => { + expect(buildScript()).toContain( + 'rclone deletefile --s3-region=auto ":s3:bucket/my-app/backup.sql.gz"', + ); + }); +}); diff --git a/packages/server/src/utils/backups/utils.ts b/packages/server/src/utils/backups/utils.ts index dba8439564..d5be0e37db 100644 --- a/packages/server/src/utils/backups/utils.ts +++ b/packages/server/src/utils/backups/utils.ts @@ -265,6 +265,12 @@ export const getBackupCommand = ( const containerSearch = getContainerSearchCommand(backup); const backupCommand = generateBackupCommand(backup); + // A failed stream can leave a truncated object behind in the bucket, + // so derive the matching delete command for the failure paths. + const rcloneCleanupCommand = rcloneCommand.startsWith("rclone rcat ") + ? `${rcloneCommand.replace("rclone rcat ", "rclone deletefile ")} >/dev/null 2>&1 || true;` + : ""; + logger.info( { containerSearch, @@ -279,7 +285,7 @@ export const getBackupCommand = ( set -eo pipefail; echo "[$(date)] Starting backup process..." >> ${logPath}; echo "[$(date)] Executing backup command..." >> ${logPath}; - CONTAINER_ID=$(${containerSearch}) + CONTAINER_ID=$(${containerSearch}); if [ -z "$CONTAINER_ID" ]; then echo "[$(date)] ❌ Error: Container not found" >> ${logPath}; @@ -288,23 +294,34 @@ export const getBackupCommand = ( echo "[$(date)] Container Up: $CONTAINER_ID" >> ${logPath}; - # Run the backup command and capture the exit status - BACKUP_OUTPUT=$(${backupCommand} 2>&1 >/dev/null) || { + echo "[$(date)] Streaming backup to S3..." >> ${logPath}; + + # Run the dump once and stream it directly into rclone. The dump exit + # code and stderr are captured through temp files so a dump failure and + # an upload failure remain distinguishable in the logs. + DUMP_STATUS_FILE=$(mktemp); + DUMP_ERROR_FILE=$(mktemp); + trap 'rm -f "$DUMP_STATUS_FILE" "$DUMP_ERROR_FILE"' EXIT; + + UPLOAD_FAILED=0; + UPLOAD_OUTPUT=$({ ${backupCommand} 2> "$DUMP_ERROR_FILE"; echo $? > "$DUMP_STATUS_FILE"; } | ${rcloneCommand} 2>&1 >/dev/null) || UPLOAD_FAILED=1; + DUMP_STATUS=$(cat "$DUMP_STATUS_FILE"); + + if [ "$DUMP_STATUS" != "0" ]; then echo "[$(date)] ❌ Error: Backup failed" >> ${logPath}; - echo "Error: $BACKUP_OUTPUT" >> ${logPath}; + echo "Error: $(cat "$DUMP_ERROR_FILE")" >> ${logPath}; + ${rcloneCleanupCommand} exit 1; - } - - echo "[$(date)] ✅ backup completed successfully" >> ${logPath}; - echo "[$(date)] Starting upload to S3..." >> ${logPath}; + fi - # Run the upload command and capture the exit status - UPLOAD_OUTPUT=$(${backupCommand} | ${rcloneCommand} 2>&1 >/dev/null) || { + if [ "$UPLOAD_FAILED" != "0" ]; then echo "[$(date)] ❌ Error: Upload to S3 failed" >> ${logPath}; echo "Error: $UPLOAD_OUTPUT" >> ${logPath}; + ${rcloneCleanupCommand} exit 1; - } + fi + echo "[$(date)] ✅ backup completed successfully" >> ${logPath}; echo "[$(date)] ✅ Upload to S3 completed successfully" >> ${logPath}; echo "Backup done ✅" >> ${logPath}; `;