diff --git a/crates/sbproxy-model-host/src/artifact/cache.rs b/crates/sbproxy-model-host/src/artifact/cache.rs index 1679a1cd..626a1b02 100644 --- a/crates/sbproxy-model-host/src/artifact/cache.rs +++ b/crates/sbproxy-model-host/src/artifact/cache.rs @@ -540,6 +540,51 @@ impl ArtifactCache { Ok(reclaimed) } + /// Count the unreferenced blobs and their bytes without deleting them, + /// for a `prune --dry-run` report. Mirrors the reference set that + /// [`Self::remove_unreferenced_blobs`] would delete. + pub(crate) fn unreferenced_blob_bytes(&self) -> Result<(u64, u64), ArtifactError> { + let referenced: BTreeSet<_> = self + .metadata_entries()? + .into_iter() + .flat_map(|metadata| { + metadata + .files + .into_iter() + .map(|file| file.sha256) + .collect::>() + }) + .collect(); + let directory = self.root.join("blobs/sha256"); + let mut bytes = 0u64; + let mut count = 0u64; + for entry in fs::read_dir(&directory) + .map_err(|source| io_error("list artifact blobs", &directory, source))? + { + let entry = + entry.map_err(|source| io_error("read artifact blob entry", &directory, source))?; + let path = entry.path(); + let Some(name) = path.file_name().and_then(|value| value.to_str()) else { + continue; + }; + if validate_digest(name).is_err() || referenced.contains(name) { + continue; + } + let metadata = fs::symlink_metadata(&path) + .map_err(|source| io_error("read artifact blob metadata", &path, source))?; + if metadata.file_type().is_symlink() || !metadata.is_file() { + continue; + } + bytes = bytes.checked_add(metadata.len()).ok_or_else(|| { + ArtifactError::InvalidArtifact( + "unreferenced artifact bytes overflow u64".to_string(), + ) + })?; + count += 1; + } + Ok((bytes, count)) + } + pub(crate) fn inspect(&self, digest: &str) -> Result { Ok(match self.lookup(digest)? { CacheLookup::Missing => ArtifactCacheState::Missing, diff --git a/crates/sbproxy-model-host/src/artifact/gc.rs b/crates/sbproxy-model-host/src/artifact/gc.rs index 06c9f345..f37dc102 100644 --- a/crates/sbproxy-model-host/src/artifact/gc.rs +++ b/crates/sbproxy-model-host/src/artifact/gc.rs @@ -38,6 +38,19 @@ pub struct GcReport { pub budget_unsatisfied_bytes: u64, } +/// Result of reclaiming unreferenced content-addressed blobs (WOR-1862). +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +pub struct PruneReport { + /// Physical content-addressed blob bytes before pruning. + pub before_bytes: u64, + /// Number of unreferenced blobs found (deleted unless `dry_run`). + pub orphan_blobs: u64, + /// Bytes reclaimed, or that would be reclaimed under `dry_run`. + pub reclaimed_bytes: u64, + /// Whether this was a report-only run that deleted nothing. + pub dry_run: bool, +} + /// Result of one exact, protected artifact removal. #[derive(Debug, Clone, PartialEq, Eq, Serialize)] pub struct RemoveArtifactReport { @@ -52,6 +65,33 @@ pub struct RemoveArtifactReport { } impl ArtifactManager { + /// Reclaim content-addressed blobs referenced by no cached artifact + /// (WOR-1862): orphans left by an interrupted pull or a removed + /// artifact whose shared blobs another model no longer keeps alive. + /// This is the standalone counterpart to the post-collection sweep + /// `enforce_budget` already runs. Under the collection lock so it never + /// races a concurrent pull's promote. `dry_run` reports what it would + /// reclaim and deletes nothing; a cold cache with no orphans reclaims + /// zero either way. + pub fn prune(&self, dry_run: bool) -> Result { + let _mutation = self.cache.lock_exclusive_mutation()?; + let before_bytes = self.cache.blob_bytes()?; + let (orphan_blobs, reclaimed_bytes) = if dry_run { + let (bytes, count) = self.cache.unreferenced_blob_bytes()?; + (count, bytes) + } else { + let (_, count) = self.cache.unreferenced_blob_bytes()?; + let reclaimed = self.cache.remove_unreferenced_blobs()?; + (count, reclaimed) + }; + Ok(PruneReport { + before_bytes, + orphan_blobs, + reclaimed_bytes, + dry_run, + }) + } + /// Enforce a physical blob budget without deleting protected or busy artifacts. pub fn enforce_budget( &self, diff --git a/crates/sbproxy-model-host/src/artifact/mod.rs b/crates/sbproxy-model-host/src/artifact/mod.rs index c891e6e3..864267fb 100644 --- a/crates/sbproxy-model-host/src/artifact/mod.rs +++ b/crates/sbproxy-model-host/src/artifact/mod.rs @@ -19,7 +19,7 @@ use tokio::io::AsyncWriteExt; pub use cache::{ArtifactCacheMetadata, ArtifactCacheState, ReadyArtifact}; pub(crate) use gc::explicit_protection_reason; -pub use gc::{CacheProtection, GcReport, RemoveArtifactReport}; +pub use gc::{CacheProtection, GcReport, PruneReport, RemoveArtifactReport}; #[cfg(feature = "weights")] pub use http::HttpArtifactTransport; pub use http::{ diff --git a/crates/sbproxy-model-host/src/lib.rs b/crates/sbproxy-model-host/src/lib.rs index c4c01922..bb109d2e 100644 --- a/crates/sbproxy-model-host/src/lib.rs +++ b/crates/sbproxy-model-host/src/lib.rs @@ -93,8 +93,8 @@ pub use artifact::HttpArtifactTransport; pub use artifact::{ AcquisitionContext, ArtifactCacheMetadata, ArtifactCacheState, ArtifactError, ArtifactLease, ArtifactManager, ArtifactObserver, ArtifactTransport, CacheProtection, GcReport, NetworkPolicy, - PullIntent, ReadyArtifact, ResponseDisposition, SourceCredential, TransportRequest, - TransportResponse, UnavailableArtifactTransport, + PruneReport, PullIntent, ReadyArtifact, ResponseDisposition, SourceCredential, + TransportRequest, TransportResponse, UnavailableArtifactTransport, }; pub use artifact_spec::{ AcceleratorKind, ArtifactFile, ArtifactFormat, ArtifactVariant, ComputeCapability, diff --git a/crates/sbproxy-model-host/tests/artifact_gc.rs b/crates/sbproxy-model-host/tests/artifact_gc.rs index 72a6787c..a23274a3 100644 --- a/crates/sbproxy-model-host/tests/artifact_gc.rs +++ b/crates/sbproxy-model-host/tests/artifact_gc.rs @@ -315,3 +315,51 @@ async fn locked_and_active_artifacts_are_never_collected() { Some(&"deleting".to_string()) ); } + +#[tokio::test] +async fn prune_reclaims_exactly_the_unreferenced_blobs_and_keeps_referenced_ones() { + // WOR-1862: a cold cache is a no-op; an orphan blob (referenced by no + // metadata) is reported by dry-run without deletion, then reclaimed + // exactly, while a referenced blob survives. + let directory = tempdir().unwrap(); + let manager = ArtifactManager::new(directory.path(), Arc::new(NoNetwork)).unwrap(); + + // Cold cache with no blobs: nothing to reclaim. + let cold = manager.prune(false).unwrap(); + assert_eq!(cold.orphan_blobs, 0); + assert_eq!(cold.reclaimed_bytes, 0); + + // One referenced artifact, plus an orphan blob planted directly. + let referenced = prepare_artifact(directory.path(), &manager, 'a', b"kept weights", 10).await; + let orphan_bytes = b"orphaned shard bytes"; + let orphan_digest = sha256(orphan_bytes); + let blobs = directory.path().join("blobs/sha256"); + fs::write(blobs.join(&orphan_digest), orphan_bytes).unwrap(); + + // Dry-run reports the orphan but deletes nothing. + let preview = manager.prune(true).unwrap(); + assert!(preview.dry_run); + assert_eq!(preview.orphan_blobs, 1); + assert_eq!(preview.reclaimed_bytes, orphan_bytes.len() as u64); + assert!( + blobs.join(&orphan_digest).exists(), + "dry-run must not delete" + ); + + // Real prune reclaims exactly the orphan bytes; the referenced blob stays. + let pruned = manager.prune(false).unwrap(); + assert!(!pruned.dry_run); + assert_eq!(pruned.orphan_blobs, 1); + assert_eq!(pruned.reclaimed_bytes, orphan_bytes.len() as u64); + assert!(!blobs.join(&orphan_digest).exists(), "orphan reclaimed"); + let referenced_sha = &referenced.files[0].sha256; + assert!( + blobs.join(referenced_sha).exists(), + "a referenced blob must survive prune" + ); + + // Second prune is a no-op now that the cache is clean. + let again = manager.prune(false).unwrap(); + assert_eq!(again.orphan_blobs, 0); + assert_eq!(again.reclaimed_bytes, 0); +} diff --git a/crates/sbproxy/src/main.rs b/crates/sbproxy/src/main.rs index 39966d9c..601f0f8c 100644 --- a/crates/sbproxy/src/main.rs +++ b/crates/sbproxy/src/main.rs @@ -580,6 +580,24 @@ enum ModelsSub { Lock(ModelsLockArgs), /// Check the verified local cache against the lockfile. VerifyLock(ModelsVerifyLockArgs), + /// Reclaim content-addressed blobs referenced by no cached artifact. + Prune(ModelsPruneArgs), +} + +/// `sbproxy models prune`: reclaim unreferenced weight blobs. +#[derive(clap::Args, Debug, Default)] +struct ModelsPruneArgs { + /// Cache directory to prune. Defaults to the configured + /// `proxy.model_host.cache.directory` (or legacy `serve.cache_dir`) + /// when `-f/--config` is given, then the platform default. + #[arg(long)] + cache_dir: Option, + /// Report what would be reclaimed without deleting anything. + #[arg(long)] + dry_run: bool, + /// Output format. + #[arg(long = "format", value_enum, default_value_t = OutputFormat::Text)] + format: OutputFormat, } #[derive(clap::ValueEnum, Clone, Copy, Debug, Default, PartialEq, Eq)] @@ -2366,6 +2384,7 @@ fn handle_models_subcommand( Some(ModelsSub::Remove(a)) => handle_models_remove(a, config_path), Some(ModelsSub::Ps(a)) => handle_models_ps(a), Some(ModelsSub::Stop(a)) => handle_models_stop(a), + Some(ModelsSub::Prune(a)) => handle_models_prune(a, config_path), Some(ModelsSub::Lock(a)) => handle_models_lock(a, config_path), Some(ModelsSub::VerifyLock(a)) => handle_models_verify_lock(a, config_path), } @@ -3466,6 +3485,46 @@ fn configured_artifact_protection( Ok(protection) } +fn handle_models_prune( + args: &ModelsPruneArgs, + config_path: Option<&std::path::Path>, +) -> anyhow::Result { + let configured = configured_model_cache_dir(config_path); + let root = model_cache_root(args.cache_dir.as_deref().or(configured.as_deref())); + // Prune is local-only: it never fetches, so no transport is wired. + let manager = sbproxy_model_host::ArtifactManager::new( + root, + std::sync::Arc::new(sbproxy_model_host::UnavailableArtifactTransport), + )?; + let report = manager.prune(args.dry_run)?; + let output = models_command_envelope( + "models.prune", + serde_json::json!({ + "dry_run": report.dry_run, + "orphan_blobs": report.orphan_blobs, + "reclaimed_bytes": report.reclaimed_bytes, + "before_bytes": report.before_bytes, + }), + ); + match args.format { + OutputFormat::Json => println!("{}", serde_json::to_string_pretty(&output)?), + OutputFormat::Text => { + let verb = if report.dry_run { + "would reclaim" + } else { + "reclaimed" + }; + println!( + "{verb} {} across {} unreferenced blob(s) ({} cached before prune)", + format_cache_size(report.reclaimed_bytes), + report.orphan_blobs, + format_cache_size(report.before_bytes), + ); + } + } + Ok(0) +} + fn handle_models_remove( args: &ModelsRemoveArgs, config_path: Option<&std::path::Path>, diff --git a/docs/llms-full.txt b/docs/llms-full.txt index 1bafe0b9..d956b0d4 100644 --- a/docs/llms-full.txt +++ b/docs/llms-full.txt @@ -7,7 +7,7 @@ Pairs with `/llms.txt` (the small AI-discoverable feature catalog at `docs/llms. Regenerated by `scripts/regen-llms-full.sh`. Generated; do not hand-edit. Source: https://github.com/soapbucket/sbproxy -Generated: 2026-07-19T00:59:07Z +Generated: 2026-07-19T05:36:36Z --- @@ -25417,11 +25417,23 @@ sbproxy models remove qwen2.5-0.5b-instruct \ --variant q4_k_m \ --cache-dir /var/lib/sbproxy/models \ --format json + +## Reclaim content-addressed blobs referenced by no cached artifact, +## such as orphans left by an interrupted pull. --dry-run reports the +## reclaimable bytes without deleting anything. +sbproxy models prune --cache-dir /var/lib/sbproxy/models --dry-run +sbproxy models prune --cache-dir /var/lib/sbproxy/models ``` +Because the weight store is content-addressed, two models that share a shard +store it once, and `prune` reclaims only blobs that no cached artifact still +references, so a shared blob survives while its last reference remains. Prune +runs under the same collection lock as the cache-budget sweep, so it never +races a concurrent pull. + Every JSON command uses `schema_version: 1` and a stable command name such as -`models.pull` or `models.remove`. Pull and removal results include durable job -IDs when a mutation occurred. +`models.pull`, `models.remove`, or `models.prune`. Pull and removal results +include durable job IDs when a mutation occurred. ## Managed engines diff --git a/docs/model-host.md b/docs/model-host.md index 81880e80..9fef17a3 100644 --- a/docs/model-host.md +++ b/docs/model-host.md @@ -855,11 +855,23 @@ sbproxy models remove qwen2.5-0.5b-instruct \ --variant q4_k_m \ --cache-dir /var/lib/sbproxy/models \ --format json + +# Reclaim content-addressed blobs referenced by no cached artifact, +# such as orphans left by an interrupted pull. --dry-run reports the +# reclaimable bytes without deleting anything. +sbproxy models prune --cache-dir /var/lib/sbproxy/models --dry-run +sbproxy models prune --cache-dir /var/lib/sbproxy/models ``` +Because the weight store is content-addressed, two models that share a shard +store it once, and `prune` reclaims only blobs that no cached artifact still +references, so a shared blob survives while its last reference remains. Prune +runs under the same collection lock as the cache-budget sweep, so it never +races a concurrent pull. + Every JSON command uses `schema_version: 1` and a stable command name such as -`models.pull` or `models.remove`. Pull and removal results include durable job -IDs when a mutation occurred. +`models.pull`, `models.remove`, or `models.prune`. Pull and removal results +include durable job IDs when a mutation occurred. ## Managed engines