Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -57,9 +57,10 @@ ENV PATH=/home/harperdb/.npm-global/bin:$PATH
# exit status of the cleanup commands that follow it, and `harper version` asserts
# the installed bin actually resolves and runs: v5.2.0-beta.4 published an image
# with no harper in it off a green build when npm install hit a transient ETARGET.
COPY --from=build /usr/src/harper-pro/build-tools/install-packed-harper-pro.sh /usr/src/harper-pro/build-tools/check-installed-shrinkwrap.mjs /usr/local/share/harper-pro-install/
RUN <<-EOF
set -e
npm install --global harperfast-harper-pro-*.tgz
bash /usr/local/share/harper-pro-install/install-packed-harper-pro.sh harperfast-harper-pro-*.tgz
harper version
rm harperfast-harper-pro-*.tgz
mkdir -p /home/harperdb/harper
Expand Down
3 changes: 2 additions & 1 deletion Dockerfile-gpu
Original file line number Diff line number Diff line change
Expand Up @@ -91,9 +91,10 @@ ENV PATH=/home/harperdb/.npm-global/bin:$PATH

# Install Harper Pro globally. See Dockerfile for why `set -e` and the assertion
# are load-bearing here.
COPY --from=build /usr/src/harper-pro/build-tools/install-packed-harper-pro.sh /usr/src/harper-pro/build-tools/check-installed-shrinkwrap.mjs /usr/local/share/harper-pro-install/
RUN <<-EOF
set -e
npm install --global harperfast-harper-pro-*.tgz
bash /usr/local/share/harper-pro-install/install-packed-harper-pro.sh harperfast-harper-pro-*.tgz
harper version
rm harperfast-harper-pro-*.tgz
mkdir -p /home/harperdb/harper
Expand Down
3 changes: 2 additions & 1 deletion Dockerfile-openshift
Original file line number Diff line number Diff line change
Expand Up @@ -64,10 +64,11 @@ WORKDIR /home/harperdb

# Install Harper Pro globally from the tarball produced in the build stage.
COPY --from=build /usr/src/harper-pro/harperfast-harper-pro-*.tgz .
COPY --from=build /usr/src/harper-pro/build-tools/install-packed-harper-pro.sh /usr/src/harper-pro/build-tools/check-installed-shrinkwrap.mjs /usr/local/share/harper-pro-install/
# See Dockerfile for why `set -e` and the assertion are load-bearing here.
RUN <<-EOF
set -e
npm install --global harperfast-harper-pro-*.tgz
bash /usr/local/share/harper-pro-install/install-packed-harper-pro.sh harperfast-harper-pro-*.tgz
harper version
npm cache clean --force
rm -f harperfast-harper-pro-*.tgz
Expand Down
70 changes: 70 additions & 0 deletions build-tools/check-installed-shrinkwrap.mjs
Original file line number Diff line number Diff line change
@@ -0,0 +1,70 @@
#!/usr/bin/env node
// Fail unless the dependency tree installed in a package directory is exactly the one its
// npm-shrinkwrap.json records: the same package paths at the same versions. A lock entry marked
// optional may be absent: npm skips it on an os/cpu/libc mismatch or a failed native build.
import { lstatSync, readdirSync, readFileSync } from 'node:fs';
import { join } from 'node:path';

const packageDir = process.argv[2];
if (!packageDir) throw new Error('usage: check-installed-shrinkwrap.mjs <package dir>');

const lock = JSON.parse(readFileSync(join(packageDir, 'npm-shrinkwrap.json'), 'utf8'));
if (!(lock.lockfileVersion >= 2) || !lock.packages?.['']) {
throw new Error(`unsupported npm-shrinkwrap.json: expected lockfileVersion >= 2 with a "packages" map`);
}

const installed = new Map();
const problems = [];

function collect(nodeModulesKey) {
let entries;
try {
entries = readdirSync(join(packageDir, nodeModulesKey), { withFileTypes: true });
} catch (error) {
if (error.code === 'ENOENT') return;
throw error;
}
for (const entry of entries) {
if (entry.name.startsWith('.')) continue;
if (entry.name.startsWith('@') && entry.isDirectory()) {
for (const scoped of readdirSync(join(packageDir, nodeModulesKey, entry.name))) {
collectPackage(`${nodeModulesKey}/${entry.name}/${scoped}`);
}
} else {
Comment thread
kriszyp marked this conversation as resolved.
Resolved
collectPackage(`${nodeModulesKey}/${entry.name}`);
}
}
}

function collectPackage(key) {
if (!lstatSync(join(packageDir, key)).isDirectory()) {
problems.push(`not a package directory: ${key}`);
return;
}
const { version } = JSON.parse(readFileSync(join(packageDir, key, 'package.json'), 'utf8'));
installed.set(key, version);
collect(`${key}/node_modules`);
Comment thread
kriszyp marked this conversation as resolved.
Resolved
}

collect('node_modules');

for (const [key, version] of installed) {
const expected = lock.packages[key];
if (!expected) problems.push(`extra: ${key}@${version}`);
else if (expected.version !== version)
problems.push(`version: ${key} is ${version}, shrinkwrap has ${expected.version}`);
}
const skippedOptional = [];
for (const [key, expected] of Object.entries(lock.packages)) {
if (key === '' || installed.has(key)) continue;
if (expected.optional || expected.devOptional) skippedOptional.push(key);
else problems.push(`missing: ${key}@${expected.version}`);
}

if (skippedOptional.length > 0) console.log(`Optional, not installed: ${skippedOptional.join(', ')}`);
if (problems.length > 0) {
console.error(`Installed dependencies differ from npm-shrinkwrap.json in ${packageDir}:`);
for (const problem of problems) console.error(` ${problem}`);
process.exit(1);
}
console.log(`${installed.size} installed packages match npm-shrinkwrap.json`);
17 changes: 17 additions & 0 deletions build-tools/install-packed-harper-pro.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
#!/usr/bin/env bash
# Install a packed harper-pro tarball into the global npm prefix with exactly the dependency tree its
# npm-shrinkwrap.json records. `npm install --global <tgz>` would ignore the shrinkwrap inside a local
# tarball and re-resolve every range. The shrinkwrap has devDependencies pruned
# (prune-shrinkwrap-dev.mjs) while the packed package.json still lists them, which `npm ci` rejects,
# so they are dropped for the install and the published manifest is restored afterwards.
set -euo pipefail

tgz=$(realpath "$1")
pkg="${NPM_CONFIG_PREFIX:?}/lib/node_modules/@harperfast/harper-pro"

mkdir -p "$pkg"
tar -xzf "$tgz" -C "$pkg" --strip-components=1
(cd "$pkg" && npm pkg delete devDependencies && npm ci --omit=dev --no-audit --no-fund)
tar -xzf "$tgz" -C "$pkg" --strip-components=1 package/package.json
npm rebuild --global --ignore-scripts --bin-links=true @harperfast/harper-pro
node "$(dirname "$0")/check-installed-shrinkwrap.mjs" "$pkg"
103 changes: 103 additions & 0 deletions unitTests/build-tools/checkInstalledShrinkwrap.test.mjs
Original file line number Diff line number Diff line change
@@ -0,0 +1,103 @@
import assert from 'node:assert';
import { spawnSync } from 'node:child_process';
import { mkdir, mkdtemp, rm, symlink, writeFile } from 'node:fs/promises';
import { tmpdir } from 'node:os';
import { dirname, join, resolve } from 'node:path';
import { fileURLToPath } from 'node:url';

const checker = resolve(dirname(fileURLToPath(import.meta.url)), '../../build-tools/check-installed-shrinkwrap.mjs');

const shrinkwrapPackages = {
'': { name: 'pkg', version: '1.0.0' },
'node_modules/msgpackr': { version: '2.0.4' },
'node_modules/@scope/lib': { version: '1.2.3' },
'node_modules/@scope/lib/node_modules/msgpackr': { version: '1.11.0' },
'node_modules/fsevents': { version: '2.3.3', optional: true },
'node_modules/babel-jest': { version: '29.7.0', devOptional: true },
};

describe('check-installed-shrinkwrap', function () {
let packageDir;

beforeEach(async function () {
packageDir = await mkdtemp(join(tmpdir(), 'harper-pro-installed-shrinkwrap-'));
await writeFile(
join(packageDir, 'npm-shrinkwrap.json'),
JSON.stringify({ lockfileVersion: 3, packages: shrinkwrapPackages })
);
await install('node_modules/msgpackr', '2.0.4');
await install('node_modules/@scope/lib', '1.2.3');
await install('node_modules/@scope/lib/node_modules/msgpackr', '1.11.0');
});

afterEach(async function () {
await rm(packageDir, { recursive: true, force: true });
});
Comment thread
kriszyp marked this conversation as resolved.
Resolved

async function install(key, version) {
await mkdir(join(packageDir, key), { recursive: true });
await writeFile(join(packageDir, key, 'package.json'), JSON.stringify({ version }));
}

function check() {
return spawnSync(process.execPath, [checker, packageDir], { encoding: 'utf8' });
}

it('passes when the tree matches, allowing absent optional entries', function () {
const result = check();
assert.strictEqual(result.status, 0, result.stderr);
assert.match(result.stdout, /3 installed packages match/);
assert.match(result.stdout, /node_modules\/fsevents, node_modules\/babel-jest/);
});

for (const [description, key, version] of [
['a direct dependency', 'node_modules/msgpackr', '2.1.2'],
['a scoped dependency', 'node_modules/@scope/lib', '1.3.0'],
['a nested dependency', 'node_modules/@scope/lib/node_modules/msgpackr', '1.11.5'],
['a present optional dependency', 'node_modules/fsevents', '2.3.2'],
]) {
it(`fails when ${description} drifts`, async function () {
await install(key, version);
const result = check();
assert.strictEqual(result.status, 1);
assert.match(result.stderr, new RegExp(`version: ${key} is ${version.replaceAll('.', '\\.')}`));
});
}

it('fails on a package the shrinkwrap does not list', async function () {
await install('node_modules/structon', '1.1.1');
const result = check();
assert.strictEqual(result.status, 1);
assert.match(result.stderr, /extra: node_modules\/structon@1\.1\.1/);
});

it('fails on a missing required package', async function () {
await rm(join(packageDir, 'node_modules/@scope/lib/node_modules'), { recursive: true });
const result = check();
assert.strictEqual(result.status, 1);
assert.match(result.stderr, /missing: node_modules\/@scope\/lib\/node_modules\/msgpackr@1\.11\.0/);
});

it('fails on a symlinked package', async function () {
await rm(join(packageDir, 'node_modules/msgpackr'), { recursive: true });
await symlink(
join(packageDir, 'node_modules/@scope/lib/node_modules/msgpackr'),
join(packageDir, 'node_modules/msgpackr')
);
const result = check();
assert.strictEqual(result.status, 1);
assert.match(result.stderr, /not a package directory: node_modules\/msgpackr/);
});

it('fails on a malformed package manifest', async function () {
await writeFile(join(packageDir, 'node_modules/msgpackr/package.json'), '{"version":');
assert.notStrictEqual(check().status, 0);
});

it('fails on an unsupported shrinkwrap', async function () {
await writeFile(join(packageDir, 'npm-shrinkwrap.json'), JSON.stringify({ lockfileVersion: 1, dependencies: {} }));
const result = check();
assert.notStrictEqual(result.status, 0);
assert.match(result.stderr, /unsupported npm-shrinkwrap\.json/);
});
});
Loading