Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 4 additions & 1 deletion CLAUDE.md
Original file line number Diff line number Diff line change
Expand Up @@ -186,7 +186,10 @@ Configured in `package.json`:
```json
{ "perry": { "compilePackages": ["@noble/curves", "@noble/hashes"] } }
```
First-resolved directory cached in `compile_package_dirs`; subsequent imports redirect to the same copy (dedup).
Bare imports resolve from the importer outward, matching Node/Bun. Every canonical
package root reached through `compilePackages` is tracked as a separate instance,
so nested versions compile with distinct module/symbol identities; multiple links
to the same physical root still deduplicate by canonical path.

## Known Limitations

Expand Down
7 changes: 7 additions & 0 deletions changelog.d/8533-package-instance-identity.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
**Compiler: preserve installed package-instance identity during native dependency compilation (#8516).**

Perry previously keyed `compilePackages` discovery by package name. Once one copy of a package had been found, every importer was redirected to that directory—even when normal Node/Bun resolution selected a different nested installation. A dependency tree containing `dup-pkg@1` at the root and `dup-pkg@2` beneath another package therefore compiled one copy and silently bound both importers to it.

Bare package resolution now walks outward from each importer, and the compiler tracks every canonical package root in a deterministic set. Distinct installed roots receive their existing path-derived module, linker-symbol, and object-cache identities; multiple links to the same physical root still deduplicate naturally. Native-package routing remains an explicit package policy and native-addon checks now run for every resolved package instance.

The regression suite links and runs a four-module two-version fixture and verifies `top-v1 nested-v2`, plus resolver/cache tests that cover discovery-order independence.
17 changes: 8 additions & 9 deletions crates/perry/src/commands/compile/collect_modules.rs
Original file line number Diff line number Diff line change
Expand Up @@ -182,7 +182,7 @@ fn collect_module_one(
let is_perry_native = is_in_node_modules && is_in_perry_native_package(&canonical);
let is_in_compiled_pkg = ctx.aot_discovered_modules.contains(&canonical)
|| (is_in_node_modules && is_in_compile_package(&canonical, &ctx.compile_packages))
|| ctx.compile_package_dirs.values().any(|dir| {
|| ctx.compile_package_dirs.iter().any(|dir| {
if canonical.starts_with(dir) {
// Exclude nested node_modules/ inside the compiled package
// (e.g., @solana/web3.js/node_modules/bs58/ is NOT part of @solana/web3.js)
Expand Down Expand Up @@ -1257,26 +1257,25 @@ fn collect_module_one(

match kind {
ModuleKind::NativeCompiled => {
// Record compile package directory for dedup (first-found wins).
// When the same package exists in multiple nested node_modules/,
// we always resolve to the first-found copy to avoid duplicate symbols.
// Record every resolved compile-package root. Package
// identity is the canonical root, not the package name:
// nested versions remain distinct while two symlinks to
// the same physical copy canonicalize together.
let module_name = &import.source;
if !module_name.starts_with('.') && !module_name.starts_with('/') {
let (pkg_name, _) = parse_package_specifier(module_name);
if ctx.compile_packages.contains(&pkg_name)
&& !ctx.compile_package_dirs.contains_key(&pkg_name)
{
if ctx.compile_packages.contains(&pkg_name) {
if let Some(pkg_dir) =
extract_compile_package_dir(&resolved_path, &pkg_name)
{
ctx.compile_package_dirs.insert(pkg_name, pkg_dir);
ctx.compile_package_dirs.insert(pkg_dir);
} else {
// Symlinked local package: canonical path is outside node_modules.
// Walk up from resolved_path to find the package root (dir with package.json).
let mut dir = resolved_path.parent();
while let Some(d) = dir {
if d.join("package.json").exists() {
ctx.compile_package_dirs.insert(pkg_name, d.to_path_buf());
ctx.compile_package_dirs.insert(d.to_path_buf());
break;
}
dir = d.parent();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@ fn package_root_for_compile_package(
path: &std::path::Path,
) -> Option<PathBuf> {
ctx.compile_package_dirs
.values()
.iter()
.filter(|dir| path.starts_with(dir))
.max_by_key(|dir| dir.components().count())
.cloned()
Expand Down
1 change: 0 additions & 1 deletion crates/perry/src/commands/compile/collect_modules/tests.rs
Original file line number Diff line number Diff line change
Expand Up @@ -508,7 +508,6 @@ fn guard_compile_package(
let mut ctx = CompilationContext::new(root.to_path_buf());
ctx.compile_packages.insert(package_name.to_string());
ctx.compile_package_dirs.insert(
package_name.to_string(),
root.join("node_modules")
.join(package_name)
.canonicalize()
Expand Down
128 changes: 16 additions & 112 deletions crates/perry/src/commands/compile/resolve.rs
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@

use anyhow::{anyhow, Result};
use perry_hir::ModuleKind;
use std::collections::{HashMap, HashSet};
use std::collections::{BTreeSet, HashMap, HashSet};
use std::fs;
use std::path::{Component, Path, PathBuf};
use std::sync::{Mutex, OnceLock};
Expand Down Expand Up @@ -131,10 +131,10 @@ mod bun_store_tests;
#[cfg(test)]
mod declaration_map_source_tests;
#[cfg(test)]
mod dedup_version_tests;
#[cfg(test)]
mod extension_resolution_tests;
#[cfg(test)]
mod package_instance_tests;
#[cfg(test)]
mod tests;

/// Packages that Perry provides built-in native extensions for.
Expand Down Expand Up @@ -192,83 +192,6 @@ pub(super) fn extract_compile_package_dir(
.map(Path::to_path_buf)
}

/// Read a package directory's declared `version`, if it has a readable
/// `package.json` with a string `version` field.
fn package_json_version(package_dir: &Path) -> Option<String> {
let raw = fs::read_to_string(package_dir.join("package.json")).ok()?;
let value: serde_json::Value = serde_json::from_str(&raw).ok()?;
value
.get("version")
.and_then(|v| v.as_str())
.map(str::to_string)
}

/// Whether `chosen` is a *different* copy from `found` with a *different*
/// declared version. Identical versions are a genuine duplicate install and
/// collapsing them is intended; differing versions mean the build silently
/// dropped one of them.
pub(super) fn dedup_collapses_distinct_versions(chosen: &Path, found: &Path) -> bool {
if chosen == found {
return false;
}
match (package_json_version(chosen), package_json_version(found)) {
(Some(a), Some(b)) => a != b,
// A copy with no readable version can't be proven distinct; stay
// quiet rather than warning on every unversioned local link.
_ => false,
}
}

/// The copy plain Node resolution would have picked for `package_name` as
/// imported from `importer_path`: the nearest ancestor `node_modules` that
/// holds the package. Perry's compile-package path deliberately searches the
/// project root first instead (see `search_paths` in `resolve_import`), so
/// the two can disagree.
pub(super) fn node_nearest_package_dir(
package_name: &str,
importer_path: &Path,
) -> Option<PathBuf> {
let start = importer_path.parent().unwrap_or(importer_path);
ancestor_node_modules_dirs(start)
.into_iter()
.map(|node_modules| node_modules.join(package_name))
.find(|candidate| candidate.is_dir())
}

/// Warn, at most once per package, when the compile-package resolution path
/// hands an importer a different *version* than Node would have. Covers both
/// the root-first search order and the `compile_package_dirs` first-found
/// dedup, since `chosen` is the directory actually used.
fn warn_on_version_shadowed_resolution(package_name: &str, chosen: &Path, importer_path: &Path) {
static WARNED: OnceLock<Mutex<HashSet<String>>> = OnceLock::new();
let Some(nearest) = node_nearest_package_dir(package_name, importer_path) else {
return;
};
if !dedup_collapses_distinct_versions(chosen, &nearest) {
return;
}
let warned = WARNED.get_or_init(|| Mutex::new(HashSet::new()));
let Ok(mut warned) = warned.lock() else {
return;
};
if !warned.insert(package_name.to_string()) {
return;
}
let chosen_version = package_json_version(chosen).unwrap_or_else(|| "?".to_string());
let nearest_version = package_json_version(&nearest).unwrap_or_else(|| "?".to_string());
eprintln!(
" warning: `{package_name}` is installed at two different versions and \
Perry compiles ONE copy per package name. `{importer}` gets \
{chosen_version} (from {chosen}); Node would have given it \
{nearest_version} (from {nearest}). Deduplicate the dependency (npm \
dedupe / a package override), or list the package explicitly in \
`perry.compilePackages` only where you want it compiled.",
importer = importer_path.display(),
chosen = chosen.display(),
nearest = nearest.display(),
);
}

/// Check if a file path is inside a package listed in compile_packages
pub(super) fn is_in_compile_package(path: &Path, compile_packages: &HashSet<String>) -> bool {
compile_packages.iter().any(|pkg_name| {
Expand Down Expand Up @@ -1432,7 +1355,7 @@ pub(super) fn resolve_import(
importer_path: &Path,
project_root: &Path,
compile_packages: &HashSet<String>,
compile_package_dirs: &HashMap<String, PathBuf>,
compile_package_dirs: &BTreeSet<PathBuf>,
) -> Option<(PathBuf, ModuleKind)> {
// Check if it's a native Rust stdlib module. Refs #665: when the user has
// explicitly opted the package into `perry.compilePackages`, they want
Expand Down Expand Up @@ -1509,7 +1432,7 @@ pub(super) fn resolve_import(
// dependent file never enters `ctx.native_modules`, and importing
// modules see `imported_classes=[]` for symbols re-exported from it.
let in_compile_pkg = is_in_compile_package(&canonical, compile_packages)
|| compile_package_dirs.values().any(|dir| {
|| compile_package_dirs.iter().any(|dir| {
if canonical.starts_with(dir) {
let relative = canonical.strip_prefix(dir).unwrap_or(canonical.as_path());
!relative.to_string_lossy().contains("node_modules/")
Expand Down Expand Up @@ -1554,14 +1477,12 @@ pub(super) fn resolve_import(
// Handle node_modules (bare specifiers)
let (package_name, subpath) = parse_package_specifier(import_source);

// For compile_packages, search project root first to prefer ESM versions
// over nested CJS copies (e.g., @solana/web3.js/node_modules/bs58 is CJS,
// but the top-level node_modules/bs58 has ESM support)
let search_paths = if compile_packages.contains(&package_name) {
[Some(project_root), importer_path.parent()]
} else {
[importer_path.parent(), Some(project_root)]
};
// Bare package imports are importer-relative. Start at the importing
// module and walk its ancestors exactly as Node/Bun do; the project root
// remains an additive fallback for lexical/canonical path edge cases.
// A compilePackages opt-in controls how the resolved copy is compiled, not
// which installed copy wins resolution.
let search_paths = [importer_path.parent(), Some(project_root)];

for start in search_paths.iter().flatten() {
for node_modules in ancestor_node_modules_dirs(start) {
Expand All @@ -1583,40 +1504,23 @@ pub(super) fn resolve_import(
}
// Packages listed in perry.compilePackages are compiled natively
if compile_packages.contains(&package_name) {
// Deduplicate: if we've already resolved this package from a
// different node_modules location, use the first-found directory
// to avoid duplicate symbols from identical package copies
let effective_dir = compile_package_dirs
.get(&package_name)
.unwrap_or(&package_dir);
// #7137 follow-up. Two mechanisms route this import away from
// the copy Node would have used: the root-first `search_paths`
// order just above (chosen for compile packages so a top-level
// ESM copy beats a nested CJS one), and this first-found
// `compile_package_dirs` dedup. Both were narrow while
// `compile_packages` held only hand-listed names — opting a
// package in was a deliberate act. The auto-compile default
// puts the WHOLE reachable graph in that set, so both now apply
// to every bare specifier in the project, and a tree carrying
// two majors of one package silently gets one of them. Report
// it when the versions actually differ.
warn_on_version_shadowed_resolution(&package_name, effective_dir, importer_path);
// Prefer TypeScript source over compiled JS
if let Some(src_entry) =
resolve_package_source_entry(effective_dir, subpath.as_deref())
resolve_package_source_entry(&package_dir, subpath.as_deref())
{
return Some((src_entry.canonicalize().ok()?, ModuleKind::NativeCompiled));
}
// Fall back to normal resolution but still mark as NativeCompiled
if let Some(fallback_entry) =
resolve_package_entry(effective_dir, subpath.as_deref())
resolve_package_entry(&package_dir, subpath.as_deref())
{
return Some((
fallback_entry.canonicalize().ok()?,
ModuleKind::NativeCompiled,
));
}
// If effective_dir failed (shouldn't happen), try the local dir
// The entry resolved above is the same package instance and is
// kept as the final fallback for unusual package metadata.
return Some((entry.canonicalize().ok()?, ModuleKind::NativeCompiled));
}
// For other node_modules packages, classify by file
Expand Down Expand Up @@ -1704,7 +1608,7 @@ pub(super) fn resolve_import(
if let Some(canonical) = tsconfig_paths::resolve_tsconfig_paths(import_source, importer_path) {
let in_compile_pkg = is_in_compile_package(&canonical, compile_packages)
|| compile_package_dirs
.values()
.iter()
.any(|dir| canonical.starts_with(dir));
let in_node_modules = canonical.to_string_lossy().contains("node_modules");
let kind = if is_js_file(&canonical) && !in_compile_pkg && in_node_modules {
Expand Down
118 changes: 0 additions & 118 deletions crates/perry/src/commands/compile/resolve/dedup_version_tests.rs

This file was deleted.

Loading
Loading