From 9ee3b4d372fdf1b59c8c53d770f3b1103b7c3b16 Mon Sep 17 00:00:00 2001 From: Joaquin Leon Alderete Date: Mon, 27 Jul 2026 01:42:13 +0000 Subject: [PATCH] fix(backups): scope retention to the files of its own database The object key of a backup is //.gz, so two backups of two databases hosted by the same service land in the same folder under names that carry no identity. keepLatestNBackups then lists that whole folder and deletes everything past its own keepLatestCount, which means the smallest keepLatestCount among the backups sharing the folder silently governs all of them: a daily backup set to keep 7 ends up with 1 file because a weekly backup of a second database in the same service keeps 2. Backup files are now named -.gz and retention only lists that database's files, so a backup can no longer delete another one's dumps. Web server backups keep their pattern: they already live in a folder of their own, keyed by the backup's appName. Note for existing installations: files written before this change do not match the new pattern and are no longer pruned automatically; they have to be removed once, by hand. Closes #4914 --- .../__test__/backups/retention-scope.test.ts | 179 ++++++++++++++++++ packages/server/src/utils/backups/compose.ts | 7 +- packages/server/src/utils/backups/index.ts | 50 +++-- packages/server/src/utils/backups/libsql.ts | 4 +- packages/server/src/utils/backups/mariadb.ts | 4 +- packages/server/src/utils/backups/mongo.ts | 4 +- packages/server/src/utils/backups/mysql.ts | 4 +- packages/server/src/utils/backups/postgres.ts | 4 +- packages/server/src/utils/backups/utils.ts | 11 ++ 9 files changed, 241 insertions(+), 26 deletions(-) create mode 100644 apps/dokploy/__test__/backups/retention-scope.test.ts diff --git a/apps/dokploy/__test__/backups/retention-scope.test.ts b/apps/dokploy/__test__/backups/retention-scope.test.ts new file mode 100644 index 0000000000..2f2255f91b --- /dev/null +++ b/apps/dokploy/__test__/backups/retention-scope.test.ts @@ -0,0 +1,179 @@ +import { execSync } from "node:child_process"; +import { + chmodSync, + existsSync, + mkdirSync, + mkdtempSync, + readFileSync, + rmSync, + writeFileSync, +} from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import type { BackupSchedule } from "@dokploy/server/services/backup"; +import type { Destination } from "@dokploy/server/services/destination"; +import { getKeepLatestNBackupsCommand } from "@dokploy/server/utils/backups"; +import { + getBackupFileName, + getBackupFilePrefix, +} from "@dokploy/server/utils/backups/utils"; +import { afterAll, beforeAll, beforeEach, describe, expect, it } from "vitest"; + +// Two databases hosted by the same service share one folder in the bucket, so +// retention must only ever delete the files of the database it runs for. The +// command is executed for real against an `rclone` stub that honours --include. +let dir: string; +let binDir: string; +let listingPath: string; +let deletedPath: string; + +const destination = { + bucket: "my-bucket", + accessKey: "key", + secretAccessKey: "secret", + region: "auto", + endpoint: "https://example.r2.cloudflarestorage.com", + provider: "Cloudflare", +} as unknown as Destination; + +const postgresBackup = (database: string, keepLatestCount: number) => + ({ + backupType: "database", + databaseType: "postgres", + database, + keepLatestCount, + prefix: "/", + destinationId: "destination-id", + postgres: { appName: "shared-postgres-service" }, + }) as unknown as BackupSchedule; + +beforeAll(() => { + dir = mkdtempSync(join(tmpdir(), "dokploy-retention-")); + binDir = join(dir, "bin"); + mkdirSync(binDir); + listingPath = join(dir, "listing"); + deletedPath = join(dir, "deleted"); + + // Minimal rclone: `lsf` prints the listing filtered by the --include glob + // (rclone's {a,b} alternation translated to bash extglob), `delete` records + // the path it was asked to remove. + const stub = join(binDir, "rclone"); + writeFileSync( + stub, + `#!/bin/bash +shopt -s extglob +mode="$1"; shift +include="" +target="" +while [ $# -gt 0 ]; do + case "$1" in + --include) include="$2"; shift 2;; + --s3-*) shift;; + *) target="$1"; shift;; + esac +done + +if [ "$mode" = "lsf" ]; then + pattern="\${include//\\{/@(}" + pattern="\${pattern//\\}/)}" + pattern="\${pattern//,/|}" + while read -r file; do + [ -z "$file" ] && continue + case "$file" in $pattern) echo "$file";; esac + done < "${listingPath}" + exit 0 +fi + +if [ "$mode" = "delete" ]; then + echo "$target" >> "${deletedPath}" + exit 0 +fi +exit 0 +`, + ); + chmodSync(stub, 0o755); +}); + +afterAll(() => { + if (existsSync(dir)) rmSync(dir, { recursive: true, force: true }); +}); + +beforeEach(() => { + if (existsSync(deletedPath)) rmSync(deletedPath); + // Newest last; both databases live in the same folder. + writeFileSync( + listingPath, + [ + "app-2026-07-20T00-00-00-000Z.sql.gz", + "app-2026-07-21T00-00-00-000Z.sql.gz", + "app-2026-07-22T00-00-00-000Z.sql.gz", + "analytics-2026-07-21T00-00-00-000Z.sql.gz", + "analytics-2026-07-22T00-00-00-000Z.sql.gz", + // rclone lsf terminates the last line too + "", + ].join("\n"), + ); +}); + +const runRetention = (backup: BackupSchedule) => { + execSync(getKeepLatestNBackupsCommand(backup, destination), { + shell: "/bin/bash", + stdio: "ignore", + env: { ...process.env, PATH: `${binDir}:${process.env.PATH}` }, + }); + + return existsSync(deletedPath) + ? readFileSync(deletedPath, "utf8") + .trim() + .split("\n") + .filter(Boolean) + // keep just the file name, the prefix is the same for all of them + .map((path) => path.split("/").pop() as string) + : []; +}; + +describe("getBackupFileName", () => { + it("carries the database the dump came from", () => { + expect(getBackupFileName("app", "sql.gz")).toMatch(/^app-.*\.sql\.gz$/); + expect(getBackupFileName("app", "bson.gz")).toMatch(/^app-.*\.bson\.gz$/); + }); + + it("keeps the prefix safe for an S3 key and for the rclone glob", () => { + expect(getBackupFilePrefix("my db*{x}")).toBe("my_db__x_-"); + }); +}); + +describe("keepLatestNBackups scoping", () => { + it("only deletes the files of its own database", () => { + expect(runRetention(postgresBackup("app", 2))).toEqual([ + "app-2026-07-20T00-00-00-000Z.sql.gz", + ]); + }); + + it("leaves the other database alone even with a smaller keepLatestCount", () => { + const deleted = runRetention(postgresBackup("analytics", 1)); + + expect(deleted).toEqual(["analytics-2026-07-21T00-00-00-000Z.sql.gz"]); + expect(deleted.some((file) => file.startsWith("app-"))).toBe(false); + }); + + it("deletes nothing when the database has fewer files than keepLatestCount", () => { + expect(runRetention(postgresBackup("analytics", 5))).toEqual([]); + }); + + it("still matches every backup file of a web server backup", () => { + const command = getKeepLatestNBackupsCommand( + { + backupType: "database", + databaseType: "web-server", + database: "dokploy", + keepLatestCount: 4, + prefix: "/", + appName: "web-server-backup", + } as unknown as BackupSchedule, + destination, + ); + + expect(command).toContain('--include "*.zip"'); + }); +}); diff --git a/packages/server/src/utils/backups/compose.ts b/packages/server/src/utils/backups/compose.ts index 296498ff37..c8fa55ec7c 100644 --- a/packages/server/src/utils/backups/compose.ts +++ b/packages/server/src/utils/backups/compose.ts @@ -11,7 +11,7 @@ import { sendDatabaseBackupNotifications } from "../notifications/database-backu import { execAsync, execAsyncRemote } from "../process/execAsync"; import { getBackupCommand, - getBackupTimestamp, + getBackupFileName, getS3Credentials, normalizeS3Path, } from "./utils"; @@ -25,7 +25,10 @@ export const runComposeBackup = async ( const project = await findProjectById(environment.projectId); const { prefix, databaseType, serviceName } = backup; const destination = await findDestinationById(backup.destinationId); - const backupFileName = `${getBackupTimestamp()}.${databaseType === "mongo" ? "bson" : "sql"}.gz`; + const backupFileName = getBackupFileName( + backup.database, + databaseType === "mongo" ? "bson.gz" : "sql.gz", + ); const s3AppName = serviceName ? `${appName}_${serviceName}` : appName; const bucketDestination = `${s3AppName}/${normalizeS3Path(prefix)}${backupFileName}`; const deployment = await createDeploymentBackup({ diff --git a/packages/server/src/utils/backups/index.ts b/packages/server/src/utils/backups/index.ts index d0d3045587..2c675ad5a0 100644 --- a/packages/server/src/utils/backups/index.ts +++ b/packages/server/src/utils/backups/index.ts @@ -1,6 +1,7 @@ import { CLEANUP_CRON_JOB } from "@dokploy/server/constants"; import { member } from "@dokploy/server/db/schema"; import type { BackupSchedule } from "@dokploy/server/services/backup"; +import type { Destination } from "@dokploy/server/services/destination"; import { findDestinationById } from "@dokploy/server/services/destination"; import { getAllServers } from "@dokploy/server/services/server"; import { getWebServerSettings } from "@dokploy/server/services/web-server-settings"; @@ -12,7 +13,12 @@ import { cleanupAll } from "../docker/utils"; import { sendDockerCleanupNotifications } from "../notifications/docker-cleanup"; import { execAsync, execAsyncRemote } from "../process/execAsync"; import { redactRcloneCredentials } from "./redact"; -import { getS3Credentials, normalizeS3Path, scheduleBackup } from "./utils"; +import { + getBackupFilePrefix, + getS3Credentials, + normalizeS3Path, + scheduleBackup, +} from "./utils"; export const initCronJobs = async () => { console.log("Setting up cron jobs...."); @@ -124,6 +130,34 @@ const getServiceAppName = (backup: BackupSchedule): string => { return serviceAppName || backup.appName; }; +export const getKeepLatestNBackupsCommand = ( + backup: BackupSchedule, + destination: Destination, +) => { + const rcloneFlags = getS3Credentials(destination); + const appName = getServiceAppName(backup); + const backupFilesPath = `:s3:${destination.bucket}/${appName}/${normalizeS3Path(backup.prefix)}`; + + // The include pattern makes sure nothing other than this backup's own files + // is touched by rclone: the extension keeps foreign files out, and the + // database prefix keeps out the backups of the other databases hosted by the + // same service, which share this folder. Web server backups get a folder of + // their own (keyed by the backup's appName), so they cannot collide. + const includePattern = + backup.databaseType === "web-server" + ? "*.zip" + : `${getBackupFilePrefix(backup.database)}*.{sql.gz,bson.gz}`; + + const rcloneList = `rclone lsf ${rcloneFlags.join(" ")} --include "${includePattern}" ${backupFilesPath}`; + // when we pipe the above command with this one, we only get the list of files we want to delete + const sortAndPickUnwantedBackups = `sort -r | tail -n +$((${backup.keepLatestCount}+1)) | xargs -I{}`; + // this command deletes the files + // to test the deletion before actually deleting we can add --dry-run before ${backupFilesPath}{} + const rcloneDelete = `rclone delete ${rcloneFlags.join(" ")} ${backupFilesPath}{}`; + + return `${rcloneList} | ${sortAndPickUnwantedBackups} ${rcloneDelete}`; +}; + export const keepLatestNBackups = async ( backup: BackupSchedule, serverId?: string | null, @@ -134,19 +168,7 @@ export const keepLatestNBackups = async ( try { const destination = await findDestinationById(backup.destinationId); - const rcloneFlags = getS3Credentials(destination); - const appName = getServiceAppName(backup); - const backupFilesPath = `:s3:${destination.bucket}/${appName}/${normalizeS3Path(backup.prefix)}`; - - // --include "*.bson.gz" or "*.sql.gz" or "*.zip" ensures nothing else other than the dokploy backup files are touched by rclone - const rcloneList = `rclone lsf ${rcloneFlags.join(" ")} --include "*${backup.databaseType === "web-server" ? ".zip" : ".{sql.gz,bson.gz}"}" ${backupFilesPath}`; - // when we pipe the above command with this one, we only get the list of files we want to delete - const sortAndPickUnwantedBackups = `sort -r | tail -n +$((${backup.keepLatestCount}+1)) | xargs -I{}`; - // this command deletes the files - // to test the deletion before actually deleting we can add --dry-run before ${backupFilesPath}{} - const rcloneDelete = `rclone delete ${rcloneFlags.join(" ")} ${backupFilesPath}{}`; - - const rcloneCommand = `${rcloneList} | ${sortAndPickUnwantedBackups} ${rcloneDelete}`; + const rcloneCommand = getKeepLatestNBackupsCommand(backup, destination); if (serverId) { await execAsyncRemote(serverId, rcloneCommand); diff --git a/packages/server/src/utils/backups/libsql.ts b/packages/server/src/utils/backups/libsql.ts index 7f6c85f482..6fef8983ec 100644 --- a/packages/server/src/utils/backups/libsql.ts +++ b/packages/server/src/utils/backups/libsql.ts @@ -11,7 +11,7 @@ import { sendDatabaseBackupNotifications } from "../notifications/database-backu import { execAsync, execAsyncRemote } from "../process/execAsync"; import { getBackupCommand, - getBackupTimestamp, + getBackupFileName, getS3Credentials, normalizeS3Path, } from "./utils"; @@ -31,7 +31,7 @@ export const runLibsqlBackup = async ( }); const { prefix } = backup; const destination = await findDestinationById(backup.destinationId); - const backupFileName = `${getBackupTimestamp()}.sql.gz`; + const backupFileName = getBackupFileName(backup.database, "sql.gz"); const bucketDestination = `${appName}/${normalizeS3Path(prefix)}${backupFileName}`; try { const rcloneFlags = getS3Credentials(destination); diff --git a/packages/server/src/utils/backups/mariadb.ts b/packages/server/src/utils/backups/mariadb.ts index 121910ee80..7d33238e5b 100644 --- a/packages/server/src/utils/backups/mariadb.ts +++ b/packages/server/src/utils/backups/mariadb.ts @@ -11,7 +11,7 @@ import { sendDatabaseBackupNotifications } from "../notifications/database-backu import { execAsync, execAsyncRemote } from "../process/execAsync"; import { getBackupCommand, - getBackupTimestamp, + getBackupFileName, getS3Credentials, normalizeS3Path, } from "./utils"; @@ -25,7 +25,7 @@ export const runMariadbBackup = async ( const project = await findProjectById(environment.projectId); const { prefix } = backup; const destination = await findDestinationById(backup.destinationId); - const backupFileName = `${getBackupTimestamp()}.sql.gz`; + const backupFileName = getBackupFileName(backup.database, "sql.gz"); const bucketDestination = `${appName}/${normalizeS3Path(prefix)}${backupFileName}`; const deployment = await createDeploymentBackup({ backupId: backup.backupId, diff --git a/packages/server/src/utils/backups/mongo.ts b/packages/server/src/utils/backups/mongo.ts index 4b212d70c1..c132884baf 100644 --- a/packages/server/src/utils/backups/mongo.ts +++ b/packages/server/src/utils/backups/mongo.ts @@ -11,7 +11,7 @@ import { sendDatabaseBackupNotifications } from "../notifications/database-backu import { execAsync, execAsyncRemote } from "../process/execAsync"; import { getBackupCommand, - getBackupTimestamp, + getBackupFileName, getS3Credentials, normalizeS3Path, } from "./utils"; @@ -22,7 +22,7 @@ export const runMongoBackup = async (mongo: Mongo, backup: BackupSchedule) => { const project = await findProjectById(environment.projectId); const { prefix } = backup; const destination = await findDestinationById(backup.destinationId); - const backupFileName = `${getBackupTimestamp()}.bson.gz`; + const backupFileName = getBackupFileName(backup.database, "bson.gz"); const bucketDestination = `${appName}/${normalizeS3Path(prefix)}${backupFileName}`; const deployment = await createDeploymentBackup({ backupId: backup.backupId, diff --git a/packages/server/src/utils/backups/mysql.ts b/packages/server/src/utils/backups/mysql.ts index 8d0e6188a0..f856525fd5 100644 --- a/packages/server/src/utils/backups/mysql.ts +++ b/packages/server/src/utils/backups/mysql.ts @@ -11,7 +11,7 @@ import { sendDatabaseBackupNotifications } from "../notifications/database-backu import { execAsync, execAsyncRemote } from "../process/execAsync"; import { getBackupCommand, - getBackupTimestamp, + getBackupFileName, getS3Credentials, normalizeS3Path, } from "./utils"; @@ -22,7 +22,7 @@ export const runMySqlBackup = async (mysql: MySql, backup: BackupSchedule) => { const project = await findProjectById(environment.projectId); const { prefix } = backup; const destination = await findDestinationById(backup.destinationId); - const backupFileName = `${getBackupTimestamp()}.sql.gz`; + const backupFileName = getBackupFileName(backup.database, "sql.gz"); const bucketDestination = `${appName}/${normalizeS3Path(prefix)}${backupFileName}`; const deployment = await createDeploymentBackup({ backupId: backup.backupId, diff --git a/packages/server/src/utils/backups/postgres.ts b/packages/server/src/utils/backups/postgres.ts index e57682f39a..1eb50c3186 100644 --- a/packages/server/src/utils/backups/postgres.ts +++ b/packages/server/src/utils/backups/postgres.ts @@ -11,7 +11,7 @@ import { sendDatabaseBackupNotifications } from "../notifications/database-backu import { execAsync, execAsyncRemote } from "../process/execAsync"; import { getBackupCommand, - getBackupTimestamp, + getBackupFileName, getS3Credentials, normalizeS3Path, } from "./utils"; @@ -31,7 +31,7 @@ export const runPostgresBackup = async ( }); const { prefix } = backup; const destination = await findDestinationById(backup.destinationId); - const backupFileName = `${getBackupTimestamp()}.sql.gz`; + const backupFileName = getBackupFileName(backup.database, "sql.gz"); const bucketDestination = `${appName}/${normalizeS3Path(prefix)}${backupFileName}`; try { const rcloneFlags = getS3Credentials(destination); diff --git a/packages/server/src/utils/backups/utils.ts b/packages/server/src/utils/backups/utils.ts index dba8439564..932c8aebe1 100644 --- a/packages/server/src/utils/backups/utils.ts +++ b/packages/server/src/utils/backups/utils.ts @@ -61,6 +61,17 @@ export const removeScheduleBackup = (backupId: string) => { export const getBackupTimestamp = () => new Date().toISOString().replace(/[:.]/g, "-"); +// Backups of two databases hosted by the same service land in the same folder +// of the bucket, so the file name has to carry the database it came from: +// retention lists that folder and would otherwise delete the other backup's +// files. Restricted to characters that are safe both as an S3 key and inside +// the rclone glob that keepLatestNBackups builds from this same prefix. +export const getBackupFilePrefix = (database: string) => + `${database.replace(/[^a-zA-Z0-9._-]/g, "_")}-`; + +export const getBackupFileName = (database: string, extension: string) => + `${getBackupFilePrefix(database)}${getBackupTimestamp()}.${extension}`; + export const normalizeS3Path = (prefix: string) => { // Trim whitespace and remove leading/trailing slashes const normalizedPrefix = prefix.trim().replace(/^\/+|\/+$/g, "");