From 2adb80092be74a34c5108d5cb25236c788cb7ae8 Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Wed, 28 Oct 2026 14:58:12 -0600 Subject: [PATCH 1/2] Install the shrinkwrapped dependency tree in Docker images `npm install --global ` ignores the npm-shrinkwrap.json inside a local tarball and re-resolves every range, so the images shipped different dependency versions than npm-registry installs. The run stage of Dockerfile, Dockerfile-gpu and Dockerfile-openshift now extracts the tarball into the same global-prefix location, runs `npm ci --omit=dev` there, links the bins with `npm rebuild --global`, and fails the build if the installed tree differs from the shrinkwrap. Dispatch-Task: harper-pro-image-honour-shrinkwrap-52-51 Co-Authored-By: Claude Opus 5.5 --- Dockerfile | 13 ++- Dockerfile-gpu | 7 +- Dockerfile-openshift | 6 +- build-tools/check-installed-shrinkwrap.mjs | 77 +++++++++++++ build-tools/install-packed-harper-pro.sh | 22 ++++ .../checkInstalledShrinkwrap.test.mjs | 103 ++++++++++++++++++ 6 files changed, 218 insertions(+), 10 deletions(-) create mode 100644 build-tools/check-installed-shrinkwrap.mjs create mode 100755 build-tools/install-packed-harper-pro.sh create mode 100644 unitTests/build-tools/checkInstalledShrinkwrap.test.mjs diff --git a/Dockerfile b/Dockerfile index ba4985c39..49995050b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -53,13 +53,16 @@ COPY --from=build /usr/src/harper-pro/harperfast-harper-pro-*.tgz . ENV NPM_CONFIG_PREFIX=/home/harperdb/.npm-global ENV PATH=/home/harperdb/.npm-global/bin:$PATH -# Install Harper Pro globally. Without `set -e` a failed install is masked by the -# exit status of the cleanup commands that follow it, and `harper version` asserts -# the installed bin actually resolves and runs: v5.2.0-beta.4 published an image -# with no harper in it off a green build when npm install hit a transient ETARGET. +# Install Harper Pro globally with exactly the tree its npm-shrinkwrap.json records; +# the install script fails the build on any drift from it. Without `set -e` a failed +# install is masked by the exit status of the cleanup commands that follow it, and +# `harper version` asserts the installed bin actually resolves and runs: v5.2.0-beta.4 +# published an image with no harper in it off a green build when npm install hit a +# transient ETARGET. +COPY --from=build /usr/src/harper-pro/build-tools/install-packed-harper-pro.sh /usr/src/harper-pro/build-tools/check-installed-shrinkwrap.mjs /usr/local/share/harper-pro-install/ RUN <<-EOF set -e - npm install --global harperfast-harper-pro-*.tgz + bash /usr/local/share/harper-pro-install/install-packed-harper-pro.sh harperfast-harper-pro-*.tgz harper version rm harperfast-harper-pro-*.tgz mkdir -p /home/harperdb/harper diff --git a/Dockerfile-gpu b/Dockerfile-gpu index d3f639eac..08c9d9d25 100644 --- a/Dockerfile-gpu +++ b/Dockerfile-gpu @@ -89,11 +89,12 @@ COPY --from=build /usr/src/harper-pro/harperfast-harper-pro-*.tgz . ENV NPM_CONFIG_PREFIX=/home/harperdb/.npm-global ENV PATH=/home/harperdb/.npm-global/bin:$PATH -# Install Harper Pro globally. See Dockerfile for why `set -e` and the assertion -# are load-bearing here. +# Install Harper Pro globally with exactly the tree its npm-shrinkwrap.json records. +# See Dockerfile for why `set -e` and the assertion are load-bearing here. +COPY --from=build /usr/src/harper-pro/build-tools/install-packed-harper-pro.sh /usr/src/harper-pro/build-tools/check-installed-shrinkwrap.mjs /usr/local/share/harper-pro-install/ RUN <<-EOF set -e - npm install --global harperfast-harper-pro-*.tgz + bash /usr/local/share/harper-pro-install/install-packed-harper-pro.sh harperfast-harper-pro-*.tgz harper version rm harperfast-harper-pro-*.tgz mkdir -p /home/harperdb/harper diff --git a/Dockerfile-openshift b/Dockerfile-openshift index 1abd464a1..66cba8594 100644 --- a/Dockerfile-openshift +++ b/Dockerfile-openshift @@ -62,12 +62,14 @@ EOF WORKDIR /home/harperdb -# Install Harper Pro globally from the tarball produced in the build stage. +# Install Harper Pro globally from the tarball produced in the build stage, with +# exactly the tree its npm-shrinkwrap.json records. COPY --from=build /usr/src/harper-pro/harperfast-harper-pro-*.tgz . +COPY --from=build /usr/src/harper-pro/build-tools/install-packed-harper-pro.sh /usr/src/harper-pro/build-tools/check-installed-shrinkwrap.mjs /usr/local/share/harper-pro-install/ # See Dockerfile for why `set -e` and the assertion are load-bearing here. RUN <<-EOF set -e - npm install --global harperfast-harper-pro-*.tgz + bash /usr/local/share/harper-pro-install/install-packed-harper-pro.sh harperfast-harper-pro-*.tgz harper version npm cache clean --force rm -f harperfast-harper-pro-*.tgz diff --git a/build-tools/check-installed-shrinkwrap.mjs b/build-tools/check-installed-shrinkwrap.mjs new file mode 100644 index 000000000..24c1744a6 --- /dev/null +++ b/build-tools/check-installed-shrinkwrap.mjs @@ -0,0 +1,77 @@ +#!/usr/bin/env node +// Fail unless the dependency tree installed in a package directory is exactly the one its +// npm-shrinkwrap.json records: the same package paths at the same versions. +// +// npm ignores a dependency's shrinkwrap when installing it from a local tarball, so +// `npm install --global ` re-resolved every range in the Docker images (5.1.30 shipped +// msgpackr 2.1.2 against a 2.0.4 shrinkwrap). The images install with `npm ci` instead, and run +// this so drift fails the image build. A lock entry marked optional may be absent: npm skips it on +// an os/cpu/libc mismatch or a failed native build. +// +// Usage: node build-tools/check-installed-shrinkwrap.mjs +import { lstatSync, readdirSync, readFileSync } from 'node:fs'; +import { join } from 'node:path'; + +const packageDir = process.argv[2]; +if (!packageDir) throw new Error('usage: check-installed-shrinkwrap.mjs '); + +const lock = JSON.parse(readFileSync(join(packageDir, 'npm-shrinkwrap.json'), 'utf8')); +if (!(lock.lockfileVersion >= 2) || !lock.packages?.['']) { + throw new Error(`unsupported npm-shrinkwrap.json: expected lockfileVersion >= 2 with a "packages" map`); +} + +const installed = new Map(); +const problems = []; + +function collect(nodeModulesKey) { + let entries; + try { + entries = readdirSync(join(packageDir, nodeModulesKey), { withFileTypes: true }); + } catch (error) { + if (error.code === 'ENOENT') return; + throw error; + } + for (const entry of entries) { + if (entry.name.startsWith('.')) continue; + if (entry.name.startsWith('@') && entry.isDirectory()) { + for (const scoped of readdirSync(join(packageDir, nodeModulesKey, entry.name))) { + collectPackage(`${nodeModulesKey}/${entry.name}/${scoped}`); + } + } else { + collectPackage(`${nodeModulesKey}/${entry.name}`); + } + } +} + +function collectPackage(key) { + if (!lstatSync(join(packageDir, key)).isDirectory()) { + problems.push(`not a package directory: ${key}`); + return; + } + const { version } = JSON.parse(readFileSync(join(packageDir, key, 'package.json'), 'utf8')); + installed.set(key, version); + collect(`${key}/node_modules`); +} + +collect('node_modules'); + +for (const [key, version] of installed) { + const expected = lock.packages[key]; + if (!expected) problems.push(`extra: ${key}@${version}`); + else if (expected.version !== version) + problems.push(`version: ${key} is ${version}, shrinkwrap has ${expected.version}`); +} +const skippedOptional = []; +for (const [key, expected] of Object.entries(lock.packages)) { + if (key === '' || installed.has(key)) continue; + if (expected.optional || expected.devOptional) skippedOptional.push(key); + else problems.push(`missing: ${key}@${expected.version}`); +} + +if (skippedOptional.length > 0) console.log(`Optional, not installed: ${skippedOptional.join(', ')}`); +if (problems.length > 0) { + console.error(`Installed dependencies differ from npm-shrinkwrap.json in ${packageDir}:`); + for (const problem of problems) console.error(` ${problem}`); + process.exit(1); +} +console.log(`${installed.size} installed packages match npm-shrinkwrap.json`); diff --git a/build-tools/install-packed-harper-pro.sh b/build-tools/install-packed-harper-pro.sh new file mode 100755 index 000000000..d3d67d63e --- /dev/null +++ b/build-tools/install-packed-harper-pro.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash +# Install a packed harper-pro tarball into the global npm prefix with exactly the dependency tree its +# npm-shrinkwrap.json records, as an install of the published package from the registry would. +# +# `npm install --global ` ignores the shrinkwrap inside a local tarball and re-resolves every +# range, so this extracts the package where a global install puts it and runs `npm ci` there. The +# shrinkwrap has its devDependencies pruned (prune-shrinkwrap-dev.mjs) while the packed package.json +# still lists them, which `npm ci` rejects, so they are dropped for the install and the published +# manifest is restored afterwards. +# +# Usage: install-packed-harper-pro.sh +set -euo pipefail + +tgz=$(realpath "$1") +pkg="${NPM_CONFIG_PREFIX:?}/lib/node_modules/@harperfast/harper-pro" + +mkdir -p "$pkg" +tar -xzf "$tgz" -C "$pkg" --strip-components=1 +(cd "$pkg" && npm pkg delete devDependencies && npm ci --omit=dev --no-audit --no-fund) +tar -xzf "$tgz" -C "$pkg" --strip-components=1 package/package.json +npm rebuild --global --ignore-scripts --bin-links=true @harperfast/harper-pro +node "$(dirname "$0")/check-installed-shrinkwrap.mjs" "$pkg" diff --git a/unitTests/build-tools/checkInstalledShrinkwrap.test.mjs b/unitTests/build-tools/checkInstalledShrinkwrap.test.mjs new file mode 100644 index 000000000..7965383db --- /dev/null +++ b/unitTests/build-tools/checkInstalledShrinkwrap.test.mjs @@ -0,0 +1,103 @@ +import assert from 'node:assert'; +import { spawnSync } from 'node:child_process'; +import { mkdir, mkdtemp, rm, symlink, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { dirname, join, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const checker = resolve(dirname(fileURLToPath(import.meta.url)), '../../build-tools/check-installed-shrinkwrap.mjs'); + +const shrinkwrapPackages = { + '': { name: 'pkg', version: '1.0.0' }, + 'node_modules/msgpackr': { version: '2.0.4' }, + 'node_modules/@scope/lib': { version: '1.2.3' }, + 'node_modules/@scope/lib/node_modules/msgpackr': { version: '1.11.0' }, + 'node_modules/fsevents': { version: '2.3.3', optional: true }, + 'node_modules/babel-jest': { version: '29.7.0', devOptional: true }, +}; + +describe('check-installed-shrinkwrap', function () { + let packageDir; + + beforeEach(async function () { + packageDir = await mkdtemp(join(tmpdir(), 'harper-pro-installed-shrinkwrap-')); + await writeFile( + join(packageDir, 'npm-shrinkwrap.json'), + JSON.stringify({ lockfileVersion: 3, packages: shrinkwrapPackages }) + ); + await install('node_modules/msgpackr', '2.0.4'); + await install('node_modules/@scope/lib', '1.2.3'); + await install('node_modules/@scope/lib/node_modules/msgpackr', '1.11.0'); + }); + + afterEach(async function () { + await rm(packageDir, { recursive: true, force: true }); + }); + + async function install(key, version) { + await mkdir(join(packageDir, key), { recursive: true }); + await writeFile(join(packageDir, key, 'package.json'), JSON.stringify({ version })); + } + + function check() { + return spawnSync(process.execPath, [checker, packageDir], { encoding: 'utf8' }); + } + + it('passes when the tree matches, allowing absent optional entries', function () { + const result = check(); + assert.strictEqual(result.status, 0, result.stderr); + assert.match(result.stdout, /3 installed packages match/); + assert.match(result.stdout, /node_modules\/fsevents, node_modules\/babel-jest/); + }); + + for (const [description, key, version] of [ + ['a direct dependency', 'node_modules/msgpackr', '2.1.2'], + ['a scoped dependency', 'node_modules/@scope/lib', '1.3.0'], + ['a nested dependency', 'node_modules/@scope/lib/node_modules/msgpackr', '1.11.5'], + ['a present optional dependency', 'node_modules/fsevents', '2.3.2'], + ]) { + it(`fails when ${description} drifts`, async function () { + await install(key, version); + const result = check(); + assert.strictEqual(result.status, 1); + assert.match(result.stderr, new RegExp(`version: ${key} is ${version.replaceAll('.', '\\.')}`)); + }); + } + + it('fails on a package the shrinkwrap does not list', async function () { + await install('node_modules/structon', '1.1.1'); + const result = check(); + assert.strictEqual(result.status, 1); + assert.match(result.stderr, /extra: node_modules\/structon@1\.1\.1/); + }); + + it('fails on a missing required package', async function () { + await rm(join(packageDir, 'node_modules/@scope/lib/node_modules'), { recursive: true }); + const result = check(); + assert.strictEqual(result.status, 1); + assert.match(result.stderr, /missing: node_modules\/@scope\/lib\/node_modules\/msgpackr@1\.11\.0/); + }); + + it('fails on a symlinked package', async function () { + await rm(join(packageDir, 'node_modules/msgpackr'), { recursive: true }); + await symlink( + join(packageDir, 'node_modules/@scope/lib/node_modules/msgpackr'), + join(packageDir, 'node_modules/msgpackr') + ); + const result = check(); + assert.strictEqual(result.status, 1); + assert.match(result.stderr, /not a package directory: node_modules\/msgpackr/); + }); + + it('fails on an unreadable package manifest', async function () { + await writeFile(join(packageDir, 'node_modules/msgpackr/package.json'), '{"version":'); + assert.notStrictEqual(check().status, 0); + }); + + it('fails on an unsupported shrinkwrap', async function () { + await writeFile(join(packageDir, 'npm-shrinkwrap.json'), JSON.stringify({ lockfileVersion: 1, dependencies: {} })); + const result = check(); + assert.notStrictEqual(result.status, 0); + assert.match(result.stderr, /unsupported npm-shrinkwrap\.json/); + }); +}); From 1b4462381cf1e3529fa505d59018a9a28a08a49a Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Wed, 28 Oct 2026 15:07:25 -0600 Subject: [PATCH 2/2] Trim install-script comments to the non-obvious why Dispatch-Task: harper-pro-image-honour-shrinkwrap-52-51 Co-Authored-By: Claude Opus 5.5 --- Dockerfile | 10 ++++------ Dockerfile-gpu | 4 ++-- Dockerfile-openshift | 3 +-- build-tools/check-installed-shrinkwrap.mjs | 11 ++--------- build-tools/install-packed-harper-pro.sh | 13 ++++--------- .../build-tools/checkInstalledShrinkwrap.test.mjs | 2 +- 6 files changed, 14 insertions(+), 29 deletions(-) diff --git a/Dockerfile b/Dockerfile index 49995050b..71f7df124 100644 --- a/Dockerfile +++ b/Dockerfile @@ -53,12 +53,10 @@ COPY --from=build /usr/src/harper-pro/harperfast-harper-pro-*.tgz . ENV NPM_CONFIG_PREFIX=/home/harperdb/.npm-global ENV PATH=/home/harperdb/.npm-global/bin:$PATH -# Install Harper Pro globally with exactly the tree its npm-shrinkwrap.json records; -# the install script fails the build on any drift from it. Without `set -e` a failed -# install is masked by the exit status of the cleanup commands that follow it, and -# `harper version` asserts the installed bin actually resolves and runs: v5.2.0-beta.4 -# published an image with no harper in it off a green build when npm install hit a -# transient ETARGET. +# Install Harper Pro globally. Without `set -e` a failed install is masked by the +# exit status of the cleanup commands that follow it, and `harper version` asserts +# the installed bin actually resolves and runs: v5.2.0-beta.4 published an image +# with no harper in it off a green build when npm install hit a transient ETARGET. COPY --from=build /usr/src/harper-pro/build-tools/install-packed-harper-pro.sh /usr/src/harper-pro/build-tools/check-installed-shrinkwrap.mjs /usr/local/share/harper-pro-install/ RUN <<-EOF set -e diff --git a/Dockerfile-gpu b/Dockerfile-gpu index 08c9d9d25..116a46557 100644 --- a/Dockerfile-gpu +++ b/Dockerfile-gpu @@ -89,8 +89,8 @@ COPY --from=build /usr/src/harper-pro/harperfast-harper-pro-*.tgz . ENV NPM_CONFIG_PREFIX=/home/harperdb/.npm-global ENV PATH=/home/harperdb/.npm-global/bin:$PATH -# Install Harper Pro globally with exactly the tree its npm-shrinkwrap.json records. -# See Dockerfile for why `set -e` and the assertion are load-bearing here. +# Install Harper Pro globally. See Dockerfile for why `set -e` and the assertion +# are load-bearing here. COPY --from=build /usr/src/harper-pro/build-tools/install-packed-harper-pro.sh /usr/src/harper-pro/build-tools/check-installed-shrinkwrap.mjs /usr/local/share/harper-pro-install/ RUN <<-EOF set -e diff --git a/Dockerfile-openshift b/Dockerfile-openshift index 66cba8594..dadbbaad5 100644 --- a/Dockerfile-openshift +++ b/Dockerfile-openshift @@ -62,8 +62,7 @@ EOF WORKDIR /home/harperdb -# Install Harper Pro globally from the tarball produced in the build stage, with -# exactly the tree its npm-shrinkwrap.json records. +# Install Harper Pro globally from the tarball produced in the build stage. COPY --from=build /usr/src/harper-pro/harperfast-harper-pro-*.tgz . COPY --from=build /usr/src/harper-pro/build-tools/install-packed-harper-pro.sh /usr/src/harper-pro/build-tools/check-installed-shrinkwrap.mjs /usr/local/share/harper-pro-install/ # See Dockerfile for why `set -e` and the assertion are load-bearing here. diff --git a/build-tools/check-installed-shrinkwrap.mjs b/build-tools/check-installed-shrinkwrap.mjs index 24c1744a6..911f0545d 100644 --- a/build-tools/check-installed-shrinkwrap.mjs +++ b/build-tools/check-installed-shrinkwrap.mjs @@ -1,14 +1,7 @@ #!/usr/bin/env node // Fail unless the dependency tree installed in a package directory is exactly the one its -// npm-shrinkwrap.json records: the same package paths at the same versions. -// -// npm ignores a dependency's shrinkwrap when installing it from a local tarball, so -// `npm install --global ` re-resolved every range in the Docker images (5.1.30 shipped -// msgpackr 2.1.2 against a 2.0.4 shrinkwrap). The images install with `npm ci` instead, and run -// this so drift fails the image build. A lock entry marked optional may be absent: npm skips it on -// an os/cpu/libc mismatch or a failed native build. -// -// Usage: node build-tools/check-installed-shrinkwrap.mjs +// npm-shrinkwrap.json records: the same package paths at the same versions. A lock entry marked +// optional may be absent: npm skips it on an os/cpu/libc mismatch or a failed native build. import { lstatSync, readdirSync, readFileSync } from 'node:fs'; import { join } from 'node:path'; diff --git a/build-tools/install-packed-harper-pro.sh b/build-tools/install-packed-harper-pro.sh index d3d67d63e..7583f33c4 100755 --- a/build-tools/install-packed-harper-pro.sh +++ b/build-tools/install-packed-harper-pro.sh @@ -1,14 +1,9 @@ #!/usr/bin/env bash # Install a packed harper-pro tarball into the global npm prefix with exactly the dependency tree its -# npm-shrinkwrap.json records, as an install of the published package from the registry would. -# -# `npm install --global ` ignores the shrinkwrap inside a local tarball and re-resolves every -# range, so this extracts the package where a global install puts it and runs `npm ci` there. The -# shrinkwrap has its devDependencies pruned (prune-shrinkwrap-dev.mjs) while the packed package.json -# still lists them, which `npm ci` rejects, so they are dropped for the install and the published -# manifest is restored afterwards. -# -# Usage: install-packed-harper-pro.sh +# npm-shrinkwrap.json records. `npm install --global ` would ignore the shrinkwrap inside a local +# tarball and re-resolve every range. The shrinkwrap has devDependencies pruned +# (prune-shrinkwrap-dev.mjs) while the packed package.json still lists them, which `npm ci` rejects, +# so they are dropped for the install and the published manifest is restored afterwards. set -euo pipefail tgz=$(realpath "$1") diff --git a/unitTests/build-tools/checkInstalledShrinkwrap.test.mjs b/unitTests/build-tools/checkInstalledShrinkwrap.test.mjs index 7965383db..2c4bfdff3 100644 --- a/unitTests/build-tools/checkInstalledShrinkwrap.test.mjs +++ b/unitTests/build-tools/checkInstalledShrinkwrap.test.mjs @@ -89,7 +89,7 @@ describe('check-installed-shrinkwrap', function () { assert.match(result.stderr, /not a package directory: node_modules\/msgpackr/); }); - it('fails on an unreadable package manifest', async function () { + it('fails on a malformed package manifest', async function () { await writeFile(join(packageDir, 'node_modules/msgpackr/package.json'), '{"version":'); assert.notStrictEqual(check().status, 0); });