diff --git a/apps/dokploy/__test__/backups/retention-scope.test.ts b/apps/dokploy/__test__/backups/retention-scope.test.ts new file mode 100644 index 0000000000..2f2255f91b --- /dev/null +++ b/apps/dokploy/__test__/backups/retention-scope.test.ts @@ -0,0 +1,179 @@ +import { execSync } from "node:child_process"; +import { + chmodSync, + existsSync, + mkdirSync, + mkdtempSync, + readFileSync, + rmSync, + writeFileSync, +} from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import type { BackupSchedule } from "@dokploy/server/services/backup"; +import type { Destination } from "@dokploy/server/services/destination"; +import { getKeepLatestNBackupsCommand } from "@dokploy/server/utils/backups"; +import { + getBackupFileName, + getBackupFilePrefix, +} from "@dokploy/server/utils/backups/utils"; +import { afterAll, beforeAll, beforeEach, describe, expect, it } from "vitest"; + +// Two databases hosted by the same service share one folder in the bucket, so +// retention must only ever delete the files of the database it runs for. The +// command is executed for real against an `rclone` stub that honours --include. +let dir: string; +let binDir: string; +let listingPath: string; +let deletedPath: string; + +const destination = { + bucket: "my-bucket", + accessKey: "key", + secretAccessKey: "secret", + region: "auto", + endpoint: "https://example.r2.cloudflarestorage.com", + provider: "Cloudflare", +} as unknown as Destination; + +const postgresBackup = (database: string, keepLatestCount: number) => + ({ + backupType: "database", + databaseType: "postgres", + database, + keepLatestCount, + prefix: "/", + destinationId: "destination-id", + postgres: { appName: "shared-postgres-service" }, + }) as unknown as BackupSchedule; + +beforeAll(() => { + dir = mkdtempSync(join(tmpdir(), "dokploy-retention-")); + binDir = join(dir, "bin"); + mkdirSync(binDir); + listingPath = join(dir, "listing"); + deletedPath = join(dir, "deleted"); + + // Minimal rclone: `lsf` prints the listing filtered by the --include glob + // (rclone's {a,b} alternation translated to bash extglob), `delete` records + // the path it was asked to remove. + const stub = join(binDir, "rclone"); + writeFileSync( + stub, + `#!/bin/bash +shopt -s extglob +mode="$1"; shift +include="" +target="" +while [ $# -gt 0 ]; do + case "$1" in + --include) include="$2"; shift 2;; + --s3-*) shift;; + *) target="$1"; shift;; + esac +done + +if [ "$mode" = "lsf" ]; then + pattern="\${include//\\{/@(}" + pattern="\${pattern//\\}/)}" + pattern="\${pattern//,/|}" + while read -r file; do + [ -z "$file" ] && continue + case "$file" in $pattern) echo "$file";; esac + done < "${listingPath}" + exit 0 +fi + +if [ "$mode" = "delete" ]; then + echo "$target" >> "${deletedPath}" + exit 0 +fi +exit 0 +`, + ); + chmodSync(stub, 0o755); +}); + +afterAll(() => { + if (existsSync(dir)) rmSync(dir, { recursive: true, force: true }); +}); + +beforeEach(() => { + if (existsSync(deletedPath)) rmSync(deletedPath); + // Newest last; both databases live in the same folder. + writeFileSync( + listingPath, + [ + "app-2026-07-20T00-00-00-000Z.sql.gz", + "app-2026-07-21T00-00-00-000Z.sql.gz", + "app-2026-07-22T00-00-00-000Z.sql.gz", + "analytics-2026-07-21T00-00-00-000Z.sql.gz", + "analytics-2026-07-22T00-00-00-000Z.sql.gz", + // rclone lsf terminates the last line too + "", + ].join("\n"), + ); +}); + +const runRetention = (backup: BackupSchedule) => { + execSync(getKeepLatestNBackupsCommand(backup, destination), { + shell: "/bin/bash", + stdio: "ignore", + env: { ...process.env, PATH: `${binDir}:${process.env.PATH}` }, + }); + + return existsSync(deletedPath) + ? readFileSync(deletedPath, "utf8") + .trim() + .split("\n") + .filter(Boolean) + // keep just the file name, the prefix is the same for all of them + .map((path) => path.split("/").pop() as string) + : []; +}; + +describe("getBackupFileName", () => { + it("carries the database the dump came from", () => { + expect(getBackupFileName("app", "sql.gz")).toMatch(/^app-.*\.sql\.gz$/); + expect(getBackupFileName("app", "bson.gz")).toMatch(/^app-.*\.bson\.gz$/); + }); + + it("keeps the prefix safe for an S3 key and for the rclone glob", () => { + expect(getBackupFilePrefix("my db*{x}")).toBe("my_db__x_-"); + }); +}); + +describe("keepLatestNBackups scoping", () => { + it("only deletes the files of its own database", () => { + expect(runRetention(postgresBackup("app", 2))).toEqual([ + "app-2026-07-20T00-00-00-000Z.sql.gz", + ]); + }); + + it("leaves the other database alone even with a smaller keepLatestCount", () => { + const deleted = runRetention(postgresBackup("analytics", 1)); + + expect(deleted).toEqual(["analytics-2026-07-21T00-00-00-000Z.sql.gz"]); + expect(deleted.some((file) => file.startsWith("app-"))).toBe(false); + }); + + it("deletes nothing when the database has fewer files than keepLatestCount", () => { + expect(runRetention(postgresBackup("analytics", 5))).toEqual([]); + }); + + it("still matches every backup file of a web server backup", () => { + const command = getKeepLatestNBackupsCommand( + { + backupType: "database", + databaseType: "web-server", + database: "dokploy", + keepLatestCount: 4, + prefix: "/", + appName: "web-server-backup", + } as unknown as BackupSchedule, + destination, + ); + + expect(command).toContain('--include "*.zip"'); + }); +}); diff --git a/packages/server/src/utils/backups/compose.ts b/packages/server/src/utils/backups/compose.ts index 296498ff37..c8fa55ec7c 100644 --- a/packages/server/src/utils/backups/compose.ts +++ b/packages/server/src/utils/backups/compose.ts @@ -11,7 +11,7 @@ import { sendDatabaseBackupNotifications } from "../notifications/database-backu import { execAsync, execAsyncRemote } from "../process/execAsync"; import { getBackupCommand, - getBackupTimestamp, + getBackupFileName, getS3Credentials, normalizeS3Path, } from "./utils"; @@ -25,7 +25,10 @@ export const runComposeBackup = async ( const project = await findProjectById(environment.projectId); const { prefix, databaseType, serviceName } = backup; const destination = await findDestinationById(backup.destinationId); - const backupFileName = `${getBackupTimestamp()}.${databaseType === "mongo" ? "bson" : "sql"}.gz`; + const backupFileName = getBackupFileName( + backup.database, + databaseType === "mongo" ? "bson.gz" : "sql.gz", + ); const s3AppName = serviceName ? `${appName}_${serviceName}` : appName; const bucketDestination = `${s3AppName}/${normalizeS3Path(prefix)}${backupFileName}`; const deployment = await createDeploymentBackup({ diff --git a/packages/server/src/utils/backups/index.ts b/packages/server/src/utils/backups/index.ts index d0d3045587..2c675ad5a0 100644 --- a/packages/server/src/utils/backups/index.ts +++ b/packages/server/src/utils/backups/index.ts @@ -1,6 +1,7 @@ import { CLEANUP_CRON_JOB } from "@dokploy/server/constants"; import { member } from "@dokploy/server/db/schema"; import type { BackupSchedule } from "@dokploy/server/services/backup"; +import type { Destination } from "@dokploy/server/services/destination"; import { findDestinationById } from "@dokploy/server/services/destination"; import { getAllServers } from "@dokploy/server/services/server"; import { getWebServerSettings } from "@dokploy/server/services/web-server-settings"; @@ -12,7 +13,12 @@ import { cleanupAll } from "../docker/utils"; import { sendDockerCleanupNotifications } from "../notifications/docker-cleanup"; import { execAsync, execAsyncRemote } from "../process/execAsync"; import { redactRcloneCredentials } from "./redact"; -import { getS3Credentials, normalizeS3Path, scheduleBackup } from "./utils"; +import { + getBackupFilePrefix, + getS3Credentials, + normalizeS3Path, + scheduleBackup, +} from "./utils"; export const initCronJobs = async () => { console.log("Setting up cron jobs...."); @@ -124,6 +130,34 @@ const getServiceAppName = (backup: BackupSchedule): string => { return serviceAppName || backup.appName; }; +export const getKeepLatestNBackupsCommand = ( + backup: BackupSchedule, + destination: Destination, +) => { + const rcloneFlags = getS3Credentials(destination); + const appName = getServiceAppName(backup); + const backupFilesPath = `:s3:${destination.bucket}/${appName}/${normalizeS3Path(backup.prefix)}`; + + // The include pattern makes sure nothing other than this backup's own files + // is touched by rclone: the extension keeps foreign files out, and the + // database prefix keeps out the backups of the other databases hosted by the + // same service, which share this folder. Web server backups get a folder of + // their own (keyed by the backup's appName), so they cannot collide. + const includePattern = + backup.databaseType === "web-server" + ? "*.zip" + : `${getBackupFilePrefix(backup.database)}*.{sql.gz,bson.gz}`; + + const rcloneList = `rclone lsf ${rcloneFlags.join(" ")} --include "${includePattern}" ${backupFilesPath}`; + // when we pipe the above command with this one, we only get the list of files we want to delete + const sortAndPickUnwantedBackups = `sort -r | tail -n +$((${backup.keepLatestCount}+1)) | xargs -I{}`; + // this command deletes the files + // to test the deletion before actually deleting we can add --dry-run before ${backupFilesPath}{} + const rcloneDelete = `rclone delete ${rcloneFlags.join(" ")} ${backupFilesPath}{}`; + + return `${rcloneList} | ${sortAndPickUnwantedBackups} ${rcloneDelete}`; +}; + export const keepLatestNBackups = async ( backup: BackupSchedule, serverId?: string | null, @@ -134,19 +168,7 @@ export const keepLatestNBackups = async ( try { const destination = await findDestinationById(backup.destinationId); - const rcloneFlags = getS3Credentials(destination); - const appName = getServiceAppName(backup); - const backupFilesPath = `:s3:${destination.bucket}/${appName}/${normalizeS3Path(backup.prefix)}`; - - // --include "*.bson.gz" or "*.sql.gz" or "*.zip" ensures nothing else other than the dokploy backup files are touched by rclone - const rcloneList = `rclone lsf ${rcloneFlags.join(" ")} --include "*${backup.databaseType === "web-server" ? ".zip" : ".{sql.gz,bson.gz}"}" ${backupFilesPath}`; - // when we pipe the above command with this one, we only get the list of files we want to delete - const sortAndPickUnwantedBackups = `sort -r | tail -n +$((${backup.keepLatestCount}+1)) | xargs -I{}`; - // this command deletes the files - // to test the deletion before actually deleting we can add --dry-run before ${backupFilesPath}{} - const rcloneDelete = `rclone delete ${rcloneFlags.join(" ")} ${backupFilesPath}{}`; - - const rcloneCommand = `${rcloneList} | ${sortAndPickUnwantedBackups} ${rcloneDelete}`; + const rcloneCommand = getKeepLatestNBackupsCommand(backup, destination); if (serverId) { await execAsyncRemote(serverId, rcloneCommand); diff --git a/packages/server/src/utils/backups/libsql.ts b/packages/server/src/utils/backups/libsql.ts index 7f6c85f482..6fef8983ec 100644 --- a/packages/server/src/utils/backups/libsql.ts +++ b/packages/server/src/utils/backups/libsql.ts @@ -11,7 +11,7 @@ import { sendDatabaseBackupNotifications } from "../notifications/database-backu import { execAsync, execAsyncRemote } from "../process/execAsync"; import { getBackupCommand, - getBackupTimestamp, + getBackupFileName, getS3Credentials, normalizeS3Path, } from "./utils"; @@ -31,7 +31,7 @@ export const runLibsqlBackup = async ( }); const { prefix } = backup; const destination = await findDestinationById(backup.destinationId); - const backupFileName = `${getBackupTimestamp()}.sql.gz`; + const backupFileName = getBackupFileName(backup.database, "sql.gz"); const bucketDestination = `${appName}/${normalizeS3Path(prefix)}${backupFileName}`; try { const rcloneFlags = getS3Credentials(destination); diff --git a/packages/server/src/utils/backups/mariadb.ts b/packages/server/src/utils/backups/mariadb.ts index 121910ee80..7d33238e5b 100644 --- a/packages/server/src/utils/backups/mariadb.ts +++ b/packages/server/src/utils/backups/mariadb.ts @@ -11,7 +11,7 @@ import { sendDatabaseBackupNotifications } from "../notifications/database-backu import { execAsync, execAsyncRemote } from "../process/execAsync"; import { getBackupCommand, - getBackupTimestamp, + getBackupFileName, getS3Credentials, normalizeS3Path, } from "./utils"; @@ -25,7 +25,7 @@ export const runMariadbBackup = async ( const project = await findProjectById(environment.projectId); const { prefix } = backup; const destination = await findDestinationById(backup.destinationId); - const backupFileName = `${getBackupTimestamp()}.sql.gz`; + const backupFileName = getBackupFileName(backup.database, "sql.gz"); const bucketDestination = `${appName}/${normalizeS3Path(prefix)}${backupFileName}`; const deployment = await createDeploymentBackup({ backupId: backup.backupId, diff --git a/packages/server/src/utils/backups/mongo.ts b/packages/server/src/utils/backups/mongo.ts index 4b212d70c1..c132884baf 100644 --- a/packages/server/src/utils/backups/mongo.ts +++ b/packages/server/src/utils/backups/mongo.ts @@ -11,7 +11,7 @@ import { sendDatabaseBackupNotifications } from "../notifications/database-backu import { execAsync, execAsyncRemote } from "../process/execAsync"; import { getBackupCommand, - getBackupTimestamp, + getBackupFileName, getS3Credentials, normalizeS3Path, } from "./utils"; @@ -22,7 +22,7 @@ export const runMongoBackup = async (mongo: Mongo, backup: BackupSchedule) => { const project = await findProjectById(environment.projectId); const { prefix } = backup; const destination = await findDestinationById(backup.destinationId); - const backupFileName = `${getBackupTimestamp()}.bson.gz`; + const backupFileName = getBackupFileName(backup.database, "bson.gz"); const bucketDestination = `${appName}/${normalizeS3Path(prefix)}${backupFileName}`; const deployment = await createDeploymentBackup({ backupId: backup.backupId, diff --git a/packages/server/src/utils/backups/mysql.ts b/packages/server/src/utils/backups/mysql.ts index 8d0e6188a0..f856525fd5 100644 --- a/packages/server/src/utils/backups/mysql.ts +++ b/packages/server/src/utils/backups/mysql.ts @@ -11,7 +11,7 @@ import { sendDatabaseBackupNotifications } from "../notifications/database-backu import { execAsync, execAsyncRemote } from "../process/execAsync"; import { getBackupCommand, - getBackupTimestamp, + getBackupFileName, getS3Credentials, normalizeS3Path, } from "./utils"; @@ -22,7 +22,7 @@ export const runMySqlBackup = async (mysql: MySql, backup: BackupSchedule) => { const project = await findProjectById(environment.projectId); const { prefix } = backup; const destination = await findDestinationById(backup.destinationId); - const backupFileName = `${getBackupTimestamp()}.sql.gz`; + const backupFileName = getBackupFileName(backup.database, "sql.gz"); const bucketDestination = `${appName}/${normalizeS3Path(prefix)}${backupFileName}`; const deployment = await createDeploymentBackup({ backupId: backup.backupId, diff --git a/packages/server/src/utils/backups/postgres.ts b/packages/server/src/utils/backups/postgres.ts index e57682f39a..1eb50c3186 100644 --- a/packages/server/src/utils/backups/postgres.ts +++ b/packages/server/src/utils/backups/postgres.ts @@ -11,7 +11,7 @@ import { sendDatabaseBackupNotifications } from "../notifications/database-backu import { execAsync, execAsyncRemote } from "../process/execAsync"; import { getBackupCommand, - getBackupTimestamp, + getBackupFileName, getS3Credentials, normalizeS3Path, } from "./utils"; @@ -31,7 +31,7 @@ export const runPostgresBackup = async ( }); const { prefix } = backup; const destination = await findDestinationById(backup.destinationId); - const backupFileName = `${getBackupTimestamp()}.sql.gz`; + const backupFileName = getBackupFileName(backup.database, "sql.gz"); const bucketDestination = `${appName}/${normalizeS3Path(prefix)}${backupFileName}`; try { const rcloneFlags = getS3Credentials(destination); diff --git a/packages/server/src/utils/backups/utils.ts b/packages/server/src/utils/backups/utils.ts index dba8439564..932c8aebe1 100644 --- a/packages/server/src/utils/backups/utils.ts +++ b/packages/server/src/utils/backups/utils.ts @@ -61,6 +61,17 @@ export const removeScheduleBackup = (backupId: string) => { export const getBackupTimestamp = () => new Date().toISOString().replace(/[:.]/g, "-"); +// Backups of two databases hosted by the same service land in the same folder +// of the bucket, so the file name has to carry the database it came from: +// retention lists that folder and would otherwise delete the other backup's +// files. Restricted to characters that are safe both as an S3 key and inside +// the rclone glob that keepLatestNBackups builds from this same prefix. +export const getBackupFilePrefix = (database: string) => + `${database.replace(/[^a-zA-Z0-9._-]/g, "_")}-`; + +export const getBackupFileName = (database: string, extension: string) => + `${getBackupFilePrefix(database)}${getBackupTimestamp()}.${extension}`; + export const normalizeS3Path = (prefix: string) => { // Trim whitespace and remove leading/trailing slashes const normalizedPrefix = prefix.trim().replace(/^\/+|\/+$/g, "");