Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
156 changes: 156 additions & 0 deletions apps/dokploy/__test__/backups/redact-credentials.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -48,3 +48,159 @@ describe("redactRcloneCredentials (#4621)", () => {
expect(redacted).toContain("[REDACTED]");
});
});

// Reproduction of the leak observed in production on v0.29.13:
// getS3Credentials() emits shell-quote'd values, and shell-quote leaves
// "safe" values (hex keys, tokens without spaces) UNQUOTED. The original
// redactRcloneCredentials() only masked the double-quoted form, so those
// values leaked into structured logs and error output.
// All values below are deliberately fake.
describe("redactRcloneCredentials — unquoted/variant forms (leak reproduction)", () => {
const FAKE_ACCESS = "FAKE_R2_ACCESS_KEY_0123456789";
const FAKE_SECRET = "FAKE_R2_SECRET_KEY_ABCDEFGHIJKLMNOPQRSTUVWXYZ";

const expectClean = (output: string) => {
expect(output).not.toContain(FAKE_ACCESS);
expect(output).not.toContain(FAKE_SECRET);
};

it("redacts unquoted --flag=value (shell-quote output for hex keys)", () => {
const cmd = `rclone rcat --s3-access-key-id=${FAKE_ACCESS} --s3-secret-access-key=${FAKE_SECRET} --s3-region=auto ":s3:bucket/file.gz"`;
const redacted = redactRcloneCredentials(cmd);
expectClean(redacted);
expect(redacted).toContain("--s3-access-key-id=[REDACTED]");
expect(redacted).toContain("--s3-secret-access-key=[REDACTED]");
expect(redacted).toContain("--s3-region=auto");
});

it('redacts double-quoted --flag="value"', () => {
const cmd = `rclone rcat --s3-access-key-id="${FAKE_ACCESS}" --s3-secret-access-key="${FAKE_SECRET}" ":s3:bucket/file.gz"`;
const redacted = redactRcloneCredentials(cmd);
expectClean(redacted);
expect(redacted).toContain('--s3-access-key-id="[REDACTED]"');
expect(redacted).toContain('--s3-secret-access-key="[REDACTED]"');
});

it("redacts single-quoted --flag='value'", () => {
const cmd = `rclone rcat --s3-access-key-id='${FAKE_ACCESS}' --s3-secret-access-key='${FAKE_SECRET}' ":s3:bucket/file.gz"`;
const redacted = redactRcloneCredentials(cmd);
expectClean(redacted);
expect(redacted).toContain("--s3-access-key-id='[REDACTED]'");
});

it("redacts space-separated --flag value", () => {
const cmd = `rclone rcat --s3-access-key-id ${FAKE_ACCESS} --s3-secret-access-key ${FAKE_SECRET} ":s3:bucket/file.gz"`;
const redacted = redactRcloneCredentials(cmd);
expectClean(redacted);
expect(redacted).toContain("--s3-access-key-id [REDACTED]");
});

it("redacts space-separated quoted values", () => {
const cmd = `rclone rcat --s3-access-key-id "${FAKE_ACCESS}" --s3-secret-access-key '${FAKE_SECRET}' ":s3:bucket/file.gz"`;
const redacted = redactRcloneCredentials(cmd);
expectClean(redacted);
});

it("redacts credentials in reverse argument order", () => {
const cmd = `rclone rcat --s3-secret-access-key=${FAKE_SECRET} --s3-region=auto --s3-access-key-id=${FAKE_ACCESS} ":s3:bucket/file.gz"`;
const redacted = redactRcloneCredentials(cmd);
expectClean(redacted);
expect(redacted).toContain("--s3-region=auto");
});

it("keeps non-sensitive arguments intact", () => {
const cmd = `rclone copyto --s3-access-key-id=${FAKE_ACCESS} --s3-secret-access-key=${FAKE_SECRET} --s3-no-check-bucket --s3-force-path-style --s3-endpoint=https://fake.endpoint.example ":s3:bucket/a.tar"`;
const redacted = redactRcloneCredentials(cmd);
expectClean(redacted);
expect(redacted).toContain("--s3-no-check-bucket");
expect(redacted).toContain("--s3-force-path-style");
expect(redacted).toContain("--s3-endpoint=https://fake.endpoint.example");
expect(redacted).toContain(":s3:bucket/a.tar");
});

it("redacts credentials inside an error message echoing the command", () => {
const err = `Command execution failed: Command failed: rclone rcat --s3-access-key-id=${FAKE_ACCESS} --s3-secret-access-key=${FAKE_SECRET} ":s3:bucket/file.gz"\nERROR : s3: upload failed`;
const redacted = redactRcloneCredentials(err);
expectClean(redacted);
expect(redacted).toContain("ERROR : s3: upload failed");
});

it("redacts shell-escaped values (single-quote idiom and special chars)", () => {
// shell-quote renders a value containing a single quote as 'a'\''b'
const cmd = `rclone rcat --s3-access-key-id=${FAKE_ACCESS} --s3-secret-access-key='${FAKE_SECRET}'\\''pwned' ":s3:bucket/file.gz"`;
const redacted = redactRcloneCredentials(cmd);
expect(redacted).not.toContain(FAKE_SECRET);
expect(redacted).not.toContain("pwned");
});

it("is idempotent", () => {
const cmd = `rclone rcat --s3-access-key-id=${FAKE_ACCESS} --s3-secret-access-key="${FAKE_SECRET}" ":s3:bucket/file.gz"`;
const once = redactRcloneCredentials(cmd);
expectClean(once);
expect(redactRcloneCredentials(once)).toBe(once);
});

it("handles empty or malformed input safely", () => {
expect(redactRcloneCredentials("")).toBe("");
expect(redactRcloneCredentials("--s3-access-key-id=")).toBe(
"--s3-access-key-id=",
);
expect(redactRcloneCredentials("--s3-access-key-id")).toBe(
"--s3-access-key-id",
);
});

it("never leaves the fake secrets in the output of a full pipeline command", () => {
const cmd = `rclone lsf --s3-access-key-id=${FAKE_ACCESS} --s3-secret-access-key=${FAKE_SECRET} --include "*.sql.gz" :s3:bucket/app/ | sort -r | tail -n +31 | xargs -I{} rclone delete --s3-access-key-id=${FAKE_ACCESS} --s3-secret-access-key=${FAKE_SECRET} :s3:bucket/app/{}`;
const redacted = redactRcloneCredentials(cmd);
expectClean(redacted);
expect(redacted.match(/\[REDACTED\]/g)).toHaveLength(4);
});
});

// Truncated command strings (e.g. inside an error message) can carry an
// unterminated quoted value. The last-resort pass must still mask it.
describe("redactRcloneCredentials — malformed/truncated forms", () => {
const FAKE_ACCESS = "FAKE_R2_ACCESS_KEY_0123456789";
const FAKE_SECRET = "FAKE_R2_SECRET_KEY_ABCDEFGHIJKLMNOPQRSTUVWXYZ";

it("redacts an unterminated double-quoted value", () => {
const redacted = redactRcloneCredentials(
`rclone rcat --s3-access-key-id="${FAKE_ACCESS}`,
);
expect(redacted).not.toContain(FAKE_ACCESS);
});

it("redacts an unterminated single-quoted value", () => {
const redacted = redactRcloneCredentials(
`rclone rcat --s3-secret-access-key='${FAKE_SECRET}`,
);
expect(redacted).not.toContain(FAKE_SECRET);
});

it("redacts a value ending with an escaped quote at end of string", () => {
const redacted = redactRcloneCredentials(
`--s3-secret-access-key="${FAKE_SECRET}\\"`,
);
expect(redacted).not.toContain(FAKE_SECRET);
});

it("redacts an unterminated quoted value in space-separated form", () => {
const redacted = redactRcloneCredentials(
`rclone rcat --s3-access-key-id "${FAKE_ACCESS}`,
);
expect(redacted).not.toContain(FAKE_ACCESS);
});

it("keeps the quoted placeholder shape of balanced forms (last-resort pass is a no-op)", () => {
expect(redactRcloneCredentials(`--s3-access-key-id="${FAKE_ACCESS}"`)).toBe(
'--s3-access-key-id="[REDACTED]"',
);
expect(redactRcloneCredentials(`--s3-access-key-id='${FAKE_ACCESS}'`)).toBe(
"--s3-access-key-id='[REDACTED]'",
);
expect(redactRcloneCredentials('--s3-access-key-id="[REDACTED]"')).toBe(
'--s3-access-key-id="[REDACTED]"',
);
});
});
11 changes: 8 additions & 3 deletions apps/dokploy/server/api/routers/backup.ts
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@ import {
import { findDestinationById } from "@dokploy/server/services/destination";
import { checkServicePermissionAndAccess } from "@dokploy/server/services/permission";
import { runComposeBackup } from "@dokploy/server/utils/backups/compose";
import { redactRcloneCredentials } from "@dokploy/server/utils/backups/redact";
import {
getS3Credentials,
normalizeS3Path,
Expand Down Expand Up @@ -551,13 +552,17 @@ export const backupRouter = createTRPCRouter({

return results.slice(0, 100);
} catch (error) {
console.error("Error in listBackupFiles:", error);
console.error(
"Error in listBackupFiles:",
redactRcloneCredentials(String(error)),
);
throw new TRPCError({
code: "BAD_REQUEST",
message:
message: redactRcloneCredentials(
error instanceof Error
? error.message
: "Error listing backup files",
),
cause: error,
});
}
Expand Down Expand Up @@ -611,7 +616,7 @@ export const backupRouter = createTRPCRouter({
runRestore()
.catch((error) => {
onLog(
`Error: ${error instanceof Error ? error.message : String(error)}`,
`Error: ${redactRcloneCredentials(error instanceof Error ? error.message : String(error))}`,
);
})
.finally(() => {
Expand Down
7 changes: 5 additions & 2 deletions apps/dokploy/server/api/routers/destination.ts
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import {
updateDestinationById,
} from "@dokploy/server";
import { db } from "@dokploy/server/db";
import { redactRcloneCredentials } from "@dokploy/server/utils/backups/redact";
import { TRPCError } from "@trpc/server";
import { desc, eq } from "drizzle-orm";
import { quote } from "shell-quote";
Expand Down Expand Up @@ -94,10 +95,11 @@ export const destinationRouter = createTRPCRouter({
} catch (error) {
throw new TRPCError({
code: "BAD_REQUEST",
message:
message: redactRcloneCredentials(
error instanceof Error
? error?.message
: "Error connecting to bucket",
),
cause: error,
});
}
Expand Down Expand Up @@ -172,10 +174,11 @@ export const destinationRouter = createTRPCRouter({
} catch (error) {
throw new TRPCError({
code: "BAD_REQUEST",
message:
message: redactRcloneCredentials(
error instanceof Error
? error?.message
: "Error connecting to bucket",
),
cause: error,
});
}
Expand Down
13 changes: 10 additions & 3 deletions packages/server/src/utils/backups/compose.ts
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ import { findEnvironmentById } from "@dokploy/server/services/environment";
import { findProjectById } from "@dokploy/server/services/project";
import { sendDatabaseBackupNotifications } from "../notifications/database-backup";
import { execAsync, execAsyncRemote } from "../process/execAsync";
import { redactRcloneCredentials } from "./redact";
import {
getBackupCommand,
getBackupTimestamp,
Expand Down Expand Up @@ -63,20 +64,26 @@ export const runComposeBackup = async (

await updateDeploymentStatus(deployment.deploymentId, "done");
} catch (error) {
console.log(error);
console.log(redactRcloneCredentials(String(error)));
await sendDatabaseBackupNotifications({
applicationName: name,
projectName: project.name,
databaseType: getDatabaseType(databaseType),
type: "error",
// @ts-ignore
errorMessage: error?.message || "Error message not provided",
errorMessage: redactRcloneCredentials(
error?.message || "Error message not provided",
),
organizationId: project.organizationId,
databaseName: backup.database,
});

await updateDeploymentStatus(deployment.deploymentId, "error");
throw error;
throw new Error(
redactRcloneCredentials(
error instanceof Error ? error.message : "Error message not provided",
),
);
}
};

Expand Down
11 changes: 9 additions & 2 deletions packages/server/src/utils/backups/libsql.ts
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ import type { Libsql } from "@dokploy/server/services/libsql";
import { findProjectById } from "@dokploy/server/services/project";
import { sendDatabaseBackupNotifications } from "../notifications/database-backup";
import { execAsync, execAsyncRemote } from "../process/execAsync";
import { redactRcloneCredentials } from "./redact";
import {
getBackupCommand,
getBackupTimestamp,
Expand Down Expand Up @@ -69,13 +70,19 @@ export const runLibsqlBackup = async (
databaseType: "libsql",
type: "error",
// @ts-ignore
errorMessage: error?.message || "Error message not provided",
errorMessage: redactRcloneCredentials(
error?.message || "Error message not provided",
),
organizationId: project.organizationId,
databaseName: backup.database,
});

await updateDeploymentStatus(deployment.deploymentId, "error");

throw error;
throw new Error(
redactRcloneCredentials(
error instanceof Error ? error.message : "Error message not provided",
),
);
}
};
13 changes: 10 additions & 3 deletions packages/server/src/utils/backups/mariadb.ts
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ import type { Mariadb } from "@dokploy/server/services/mariadb";
import { findProjectById } from "@dokploy/server/services/project";
import { sendDatabaseBackupNotifications } from "../notifications/database-backup";
import { execAsync, execAsyncRemote } from "../process/execAsync";
import { redactRcloneCredentials } from "./redact";
import {
getBackupCommand,
getBackupTimestamp,
Expand Down Expand Up @@ -60,18 +61,24 @@ export const runMariadbBackup = async (
});
await updateDeploymentStatus(deployment.deploymentId, "done");
} catch (error) {
console.log(error);
console.log(redactRcloneCredentials(String(error)));
await sendDatabaseBackupNotifications({
applicationName: name,
projectName: project.name,
databaseType: "mariadb",
type: "error",
// @ts-ignore
errorMessage: error?.message || "Error message not provided",
errorMessage: redactRcloneCredentials(
error?.message || "Error message not provided",
),
organizationId: project.organizationId,
databaseName: backup.database,
});
await updateDeploymentStatus(deployment.deploymentId, "error");
throw error;
throw new Error(
redactRcloneCredentials(
error instanceof Error ? error.message : "Error message not provided",
),
);
}
};
13 changes: 10 additions & 3 deletions packages/server/src/utils/backups/mongo.ts
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ import type { Mongo } from "@dokploy/server/services/mongo";
import { findProjectById } from "@dokploy/server/services/project";
import { sendDatabaseBackupNotifications } from "../notifications/database-backup";
import { execAsync, execAsyncRemote } from "../process/execAsync";
import { redactRcloneCredentials } from "./redact";
import {
getBackupCommand,
getBackupTimestamp,
Expand Down Expand Up @@ -58,18 +59,24 @@ export const runMongoBackup = async (mongo: Mongo, backup: BackupSchedule) => {
});
await updateDeploymentStatus(deployment.deploymentId, "done");
} catch (error) {
console.log(error);
console.log(redactRcloneCredentials(String(error)));
await sendDatabaseBackupNotifications({
applicationName: name,
projectName: project.name,
databaseType: "mongodb",
type: "error",
// @ts-ignore
errorMessage: error?.message || "Error message not provided",
errorMessage: redactRcloneCredentials(
error?.message || "Error message not provided",
),
organizationId: project.organizationId,
databaseName: backup.database,
});
await updateDeploymentStatus(deployment.deploymentId, "error");
throw error;
throw new Error(
redactRcloneCredentials(
error instanceof Error ? error.message : "Error message not provided",
),
);
}
};
Loading