From 5740e2faf63f81bdb80210c710b2d4899b934e1e Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Sun, 5 Jul 2026 04:17:11 +0000 Subject: [PATCH 001/120] Implement TokenJuice pipeline and policy primitives --- README.md | 54 ++- plan/current-state-and-critique.md | 71 ++-- plan/openhuman-algorithm-port-plan.md | 10 +- plan/openhuman-integration-plan.md | 58 ++- plan/pipeline-and-ccr-plan.md | 19 +- src/cache/mod.rs | 5 +- src/cache/store.rs | 195 +++++++++- src/compress.rs | 315 +++++++++++++++- src/conversation/boundary.rs | 186 ++++++++++ src/conversation/budget.rs | 293 +++++++++++++++ src/conversation/mod.rs | 23 ++ src/conversation/tool_digest.rs | 512 ++++++++++++++++++++++++++ src/conversation/types.rs | 146 ++++++++ src/lib.rs | 21 +- src/pipeline/estimate.rs | 248 +++++++++++++ src/pipeline/mod.rs | 16 + src/pipeline/report.rs | 102 +++++ src/pipeline/transform.rs | 121 ++++++ src/policy/mod.rs | 8 + src/policy/shell.rs | 444 ++++++++++++++++++++++ src/reduce.rs | 27 +- src/reduce_tests.rs | 52 +++ src/types.rs | 21 +- 23 files changed, 2819 insertions(+), 128 deletions(-) create mode 100644 src/conversation/boundary.rs create mode 100644 src/conversation/budget.rs create mode 100644 src/conversation/mod.rs create mode 100644 src/conversation/tool_digest.rs create mode 100644 src/conversation/types.rs create mode 100644 src/pipeline/estimate.rs create mode 100644 src/pipeline/mod.rs create mode 100644 src/pipeline/report.rs create mode 100644 src/pipeline/transform.rs create mode 100644 src/policy/mod.rs create mode 100644 src/policy/shell.rs diff --git a/README.md b/README.md index c442d17..8e16f75 100644 --- a/README.md +++ b/README.md @@ -16,18 +16,17 @@ to provide a small Rust boundary where OpenHuman can route prompt, context, and conversation inputs through interchangeable compression strategies before model execution. -This repository is intentionally blank scaffolding right now. It defines the -crate shell, contribution workflow, CI/release scaffolding, and placeholder API -types. Compression algorithms, OpenHuman adapters, and benchmark claims have not -landed yet. +The crate now includes deterministic reducers, content-aware compressors, and +CCR-backed recovery primitives. The scaffold remains intentionally small while +the OpenHuman adapter migration and typed pipeline work continue. ## Direction -TinyJuice is intended to support: +TinyJuice supports and is growing toward: - provider-neutral token input compression - pluggable compression strategies -- configurable compression targets, including aggressive 80-90% reduction modes +- configurable compression targets for lossless and recoverable lossy modes - inspectable compression reports for cost, quality, and safety review - OpenHuman integration points that can be enabled without coupling the core crate to the full OpenHuman runtime @@ -56,6 +55,35 @@ Run the placeholder example: cargo run --example passthrough ``` +## API Notes + +`compress_content` and `route` return `CompressedOutput`. The `text` field is +the compatibility output ready to inline into model context. When CCR retained +an original, `body` contains the compacted body without the recovery footer and +`recovery_footer` contains the footer separately. Hosts that apply their own +post-compaction caps should truncate only `body` and then reattach +`recovery_footer` so recovery markers remain reachable. + +`compress_content_with_store` and `route_with_store` accept an injectable +`CcrStore`. Their `_report` variants also return a redacted `PipelineReport` +with byte counts, a cheap bloat estimate, applied steps, skip reason, lossy +flag, and CCR token IDs. Existing `compress_content`, `route`, and `cache::*` +helpers still use the process-global `GlobalCcrStore` for compatibility. New +tests and host adapters can use `MemoryCcrStore` to assert CCR behavior without +touching the global cache. + +Shell-producing hosts can use `apply_shell_compaction_policy` with +`ShellCompactionPolicy` before compacting command output. The default +OpenHuman-facing path uses `AllowSafeInventory`: exact file-content reads, +unsafe inventory actions, and mixed shell sequences stay raw, while repository +inventory output can still be summarized. + +Conversation-level helpers under `conversation::*` expose deterministic Hermes +primitives without host runtime dependencies: token-budget tail selection, +tool-call/result boundary alignment, latest user/assistant anchors, JSON +string-leaf shrinking, and old tool-result digesting with sensitive metadata +redaction. + Run the local analytics interface: ```sh @@ -68,8 +96,14 @@ npm run dev ```text src/ + cache/ CCR recovery stores and retrieval markers compressor/ Compression trait and input/output types + compressors/ Content-aware compressor implementations config/ Compression target and policy configuration + conversation/ Provider-neutral conversation compaction helpers + pipeline/ Typed transform/report primitives + policy/ Host compaction policy helpers + reduce/ Rule-engine reducers and command normalization openhuman/ Placeholder OpenHuman integration boundary error.rs Shared error type interface/ Self-hostable analytics UI for compression run metadata @@ -77,6 +111,8 @@ interface/ Self-hostable analytics UI for compression run metadata ## Status -TinyJuice is pre-implementation. The current `PassthroughCompressor` is a -scaffold implementation that returns input unchanged so downstream wiring can -compile while real strategies are designed and tested. +TinyJuice includes deterministic reducers, content-aware compressors, and CCR +recovery plumbing. The first typed-pipeline primitives, injectable CCR store, +report-producing router compatibility path, and cheap bloat estimators exist; +full transform conversion is still in progress. Some legacy scaffold modules +remain while the OpenHuman adapter migration continues. diff --git a/plan/current-state-and-critique.md b/plan/current-state-and-critique.md index 1df9bcb..aa20de0 100644 --- a/plan/current-state-and-critique.md +++ b/plan/current-state-and-critique.md @@ -43,10 +43,9 @@ be added as rules or rule-engine features before adding one-off compressor code. ### README Drift -`README.md` still says the repository is intentionally blank scaffolding and -that real strategies have not landed. That is no longer true. The docs should -be updated after implementation plans turn into code, but this plan does not -edit README because the user requested plan files only. +`README.md` now reflects that deterministic reducers, content-aware compressors, +and CCR recovery plumbing exist. Keep future public API contract changes +documented there or under `docs/` as implementation plans turn into code. ### Duplicate Compressor Concepts @@ -65,20 +64,32 @@ compress content when an adapter passes an extension hint such as `rs`, `py`, or normal exact reads. OpenHuman should provide a host policy that marks exact file reads as raw unless the user or tool intentionally requests a stub/summary. -### Lossy Safety Is Runtime-Checked, Not Type-Checked +### Lossy Safety Is Partly Type-Checked -`CompressOutput::lossy` is currently a flag, and `route()` is responsible for -checking CCR availability. This works, but it is too easy for future code to -accidentally bypass `route()` or return lossy text without recoverability. The -Headroom-style typed pipeline should make offload-backed transforms structurally -different from lossless reformats. +`CompressOutput::lossy` is still the compatibility flag for existing +compressors, and `route()` remains responsible for checking CCR availability. +The new `pipeline` module adds the first typed split: +`ReformatTransform` emits ordinary `TransformOutput`, while `OffloadOutput` +can only be built from a retained `CcrPutResult`. Existing compressors still +need gradual conversion into those traits before lossy safety is fully enforced +by construction. -### Global CCR Store Makes Tests And Policy Harder +### CCR Store Is Now Injectable -The global store is convenient for integration, but it makes isolated tests, -adapter-specific policies, and future backends harder. An injectable `CcrStore` -trait should be introduced while preserving the current global functions as -compatibility wrappers. +The process-global store remains for compatibility, but `CcrStore`, +`GlobalCcrStore`, and `MemoryCcrStore` now exist. `compress_content_with_store` +and `route_with_store` let tests and host adapters exercise CCR behavior +against an isolated store instead of the global cache. Remaining work is to +thread explicit stores through more OpenHuman call sites instead of relying on +the global compatibility path. + +### Pipeline Reports Are Available But Compatibility-Based + +`compress_content_with_store_report` and `route_with_store_report` now return a +redacted `PipelineReport` with byte counts, cheap bloat estimate, applied step, +CCR token ids, lossy flag, and skip reason. Because current compressors still +run through the compatibility router, the applied step is reported as +`compat_router` until compressors are converted into typed transforms. ### Missing Machine Protocol @@ -93,25 +104,22 @@ yet implement the richer policy from the references: safe inventory pipelines may compact, exact content reads stay raw, mixed shell sequences stay raw, and unsafe actions such as `find -exec` stay raw. -### Footer Placement Is A Fragile Host Contract +### Footer Placement Requires A Host Contract -`route()` appends the recovery footer to the end of the compacted text and -returns a single string. Hosts that apply their own truncation caps after -compaction (OpenHuman applies a per-tool char cap and a 16 KiB byte-cap -backstop after the TokenJuice stage) keep the head and cut the tail, which -severs the footer and leaves an unrecoverable lossy view. `CompressedOutput` -should expose the footer (or the body/footer split) as separate fields so -hosts can truncate the body and reattach the marker. Until then, every host -integration must be audited for post-compaction truncation. +`CompressedOutput` now exposes both the compatibility `text` field and separate +`body`/`recovery_footer` fields. OpenHuman's tinyagents middleware composes +those fields by truncating only the body and reattaching the footer after its +per-tool char cap and byte-cap backstop. Other host integrations still need to +follow the same contract whenever they apply post-compaction caps. -### The Rule Catalog Is Inert Without Arguments +### The Rule Catalog Depends On Arguments -`compact_output_with_policy` forwards `arguments: None, exit_code: None`, and -the rule reducer only fires for command output. OpenHuman currently calls this -minimal wrapper, so the 100-rule catalog, extension hints, query hints, and -exit-code handling are all dead at the only production call site. The plan's -Phase 1 must treat migrating that call site as the primary deliverable, not a -confirmation task. +`compact_output_with_policy` still forwards `arguments: None, exit_code: None` +for minimal call sites, but OpenHuman's tinyagents middleware now captures tool +arguments in `before_tool` and calls the full policy adapter after the tool +returns. That activates command rules, extension/query hints, and exit-code +handling on the production tinyagents path. Future host adapters should avoid +the minimal wrapper unless they genuinely lack tool metadata. ### Limited Observability @@ -143,4 +151,3 @@ status, omission counts, and fixture-backed benchmark reports. - Host installers before `reduce-json`, validation, and doctorable policy are stable. - Compression percentage marketing claims. - diff --git a/plan/openhuman-algorithm-port-plan.md b/plan/openhuman-algorithm-port-plan.md index aad81a2..787a5bb 100644 --- a/plan/openhuman-algorithm-port-plan.md +++ b/plan/openhuman-algorithm-port-plan.md @@ -59,12 +59,16 @@ the process-global cache. TinyJuice files (per pipeline-and-ccr-plan): -- `src/pipeline/mod.rs`, `src/pipeline/transform.rs`, `src/pipeline/report.rs` (NEW) -- `src/cache/store.rs` — extract `CcrStore` trait, keep global wrappers +- `src/pipeline/mod.rs`, `src/pipeline/transform.rs`, `src/pipeline/report.rs` + — present with the first transform/report primitives. +- `src/cache/store.rs` — `CcrStore`, `GlobalCcrStore`, and `MemoryCcrStore` + are present; global wrappers are preserved. OpenHuman files: -- `vendor/tinyjuice` — submodule bump (currently `4b1a34f`, behind HEAD). +- Current `../openhuman-4` has a local `src/openhuman/tokenjuice/` copy rather + than `vendor/tinyjuice`; mirror store/pipeline exports there until the crate + dependency boundary is restored. - `src/openhuman/tokenjuice/mod.rs` — re-export `CcrStore`, `PipelineReport`; `install_from_config` constructs the store explicitly instead of relying on global state, keeping the global as compatibility path. diff --git a/plan/openhuman-integration-plan.md b/plan/openhuman-integration-plan.md index 2688f50..e99e243 100644 --- a/plan/openhuman-integration-plan.md +++ b/plan/openhuman-integration-plan.md @@ -37,30 +37,22 @@ existing surface: ### Known Integration Bugs (found in review) -1. **The call site uses the minimal hook.** `middleware.rs:787` calls - `compact_output_with_policy(content, tool_name, enabled, profile)`, which - forwards `arguments: None, exit_code: None`. Consequences: command/argv - never reach the rule reducer, so the 100-rule command catalog is largely - inert in OpenHuman; extension and query hints never fire, so code/JSON/HTML - routing from file-bearing tools is disabled; exit-code-aware - failure-preserving log behavior is off. Migrating this one call site to - `compact_tool_output_with_policy(tool_name, arguments, output, exit_code, - profile)` is the single highest-value integration change. -2. **Downstream truncation can destroy recovery footers.** TinyJuice appends - its CCR footer at the *end* of compacted output. Steps 3 and 4 of the - middleware chain (per-tool char cap, byte-cap backstop) run *after* - TokenJuice and keep the head, so an output that is compacted but still over - the cap loses its footer: the model gets a lossy view with no recovery - marker while the CCR entry sits unreachable. Fix options: make the host - caps footer-aware (truncate the body, reattach the trailing marker), or - expose the footer as a separate field on `CompressedOutput` so hosts can - compose it after their own truncation. The second is the cleaner contract - and belongs in core. -3. **Vendored crate drift.** The submodule at `4b1a34f` is behind this repo's - HEAD. Any plan phase that changes footer text, marker constants, or the - hook signature must include a submodule bump plus a compatibility pass over - OpenHuman's marker parsing and tool docs (`retrieve_tool_output.rs` still - documents the legacy `retrieve_tool_output("")` sentinel). +1. **Hook migration status.** OpenHuman's tinyagents middleware now captures + `TaToolCall` arguments in `before_tool`, parses rendered shell exit-code + prefixes, and calls the full TokenJuice policy adapter after each tool + result. Command rules, extension/query hints, and exit-code-aware log + behavior are active on that production path. Keep the minimal wrapper only + for call sites that truly lack metadata. +2. **Footer-truncation status.** `CompressedOutput` now exposes separate + `body` and `recovery_footer` fields while keeping `text` as the + compatibility body+footer output. OpenHuman's tinyagents middleware caps the + body and reattaches the footer afterward. Other host-side caps still need + the same audit. +3. **OpenHuman checkout shape.** The reviewed plan assumed a + `vendor/tinyjuice` submodule, but this `../openhuman-4` checkout carries a + local TokenJuice copy under `src/openhuman/tokenjuice/`. Footer and hook + contract changes therefore need to be mirrored there until the dependency + boundary is restored. ## Current Fit @@ -105,16 +97,13 @@ TinyJuice core should own: Tasks: -- Migrate `ToolOutputMiddleware::after_tool` from `compact_output_with_policy` - to `compact_tool_output_with_policy`, passing the tool's JSON arguments and - exit code. This activates the rule catalog, extension/query hints, and - failure-preserving log behavior that are currently dead in OpenHuman. -- Bump the `vendor/tinyjuice` submodule and reconcile marker/tool-name - constants with OpenHuman's tool docs. -- Fix the footer-truncation ordering bug: either make the per-tool char cap - and byte-cap backstop footer-aware, or (preferred) split the recovery footer - into its own `CompressedOutput` field so the host truncates the body and - reattaches the footer. +- Keep `ToolOutputMiddleware::after_tool` on the full policy adapter, passing + captured tool JSON arguments and parsed exit code. +- Reconcile marker/tool-name constants with OpenHuman's tool docs. In the + current `openhuman-4` checkout, mirror crate contract changes in the local + `src/openhuman/tokenjuice/` copy rather than a `vendor/tinyjuice` submodule. +- Preserve the footer-aware cap contract: the host truncates + `CompressedOutput::body` and reattaches `CompressedOutput::recovery_footer`. - Verify credential scrubbing runs before `after_tool` compaction; if not, compaction can retain secrets in CCR (see Risks). - Document the middleware ordering invariant: handoff observes raw output @@ -241,4 +230,3 @@ Acceptance: - The host truncation caps (per-tool char cap, 16 KiB byte-cap backstop) and TinyJuice compaction currently compose blindly. Until the footer contract is fixed, any cap below the compacted size silently severs recoverability. - diff --git a/plan/pipeline-and-ccr-plan.md b/plan/pipeline-and-ccr-plan.md index ecd263c..6dd2592 100644 --- a/plan/pipeline-and-ccr-plan.md +++ b/plan/pipeline-and-ccr-plan.md @@ -88,6 +88,11 @@ Skip reasons must not include raw content. - Move token validation into reusable store helpers. - Add isolated tests using an in-memory store. +Status: implemented in the core crate. `CcrStore`, `GlobalCcrStore`, +`MemoryCcrStore`, and compatibility wrappers exist; isolated memory-store tests +cover global isolation, oversized rejection, range retrieval, and malformed +token rejection. + ### Step 2: Add Pipeline Compatibility Wrapper - Build a pipeline path that can run current compressors as compatibility @@ -95,6 +100,13 @@ Skip reasons must not include raw content. - Keep `compress_content()` and `route()` signatures intact. - Assert current behavior is unchanged through existing fixtures. +Status: partially implemented. `compress_content_with_store()` and +`route_with_store()` are store-injected compatibility paths while the old APIs +delegate to `GlobalCcrStore`. `_report` variants return `PipelineReport` with +redacted skip reasons, cheap bloat estimates, and applied-step metadata. The +typed transform traits exist, but current compressors have not been converted +into pipeline transforms yet. + ### Step 3: Convert Existing Compressors Gradually - JSON small-array table rendering becomes a reformat when no rows are omitted. @@ -120,6 +132,12 @@ Add cheap estimators for: The router can use estimators to decide whether to try transforms and to report why a transform was skipped. +Status: first pass implemented. `pipeline::estimate_bloat` reports a redacted +0-100 score plus categorical reason for JSON rows, diff context, log +repetition, search fanout, HTML markup, code body bulk, and plain-text +repetition. The router reports the estimate but does not yet use it to skip +work. + ## Acceptance Criteria - Existing public APIs continue to compile. @@ -138,4 +156,3 @@ why a transform was skipped. complete. - Do not log raw content in pipeline reports. - Do not make all compressors implement the new traits in one large PR. - diff --git a/src/cache/mod.rs b/src/cache/mod.rs index 04f22ea..367ded4 100644 --- a/src/cache/mod.rs +++ b/src/cache/mod.rs @@ -8,6 +8,7 @@ pub use marker::{ format_marker, is_recovery_tool, parse_markers, recovery_footer, }; pub use store::{ - RangeUnit, configure, disable_disk_tier, enable_disk_tier, offload, offload_checked, retrieve, - retrieve_range, short_hash, stats, + CcrPutResult, CcrStore, GlobalCcrStore, MemoryCcrStore, RangeUnit, configure, + disable_disk_tier, enable_disk_tier, offload, offload_checked, retrieve, retrieve_range, + short_hash, stats, }; diff --git a/src/cache/store.rs b/src/cache/store.rs index 98c23c0..225b664 100644 --- a/src/cache/store.rs +++ b/src/cache/store.rs @@ -29,6 +29,148 @@ pub const DEFAULT_MAX_BYTES: usize = 64 * 1024 * 1024; /// capability token. const HASH_BYTES: usize = 16; +/// Result of attempting to store an original in CCR. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct CcrPutResult { + token: String, + retained: bool, +} + +impl CcrPutResult { + /// Build a put result. Invalid token shapes are never reported as retained. + pub fn new(token: String, retained: bool) -> Self { + let retained = retained && is_valid_token(&token); + Self { token, retained } + } + + /// CCR token derived for the original. + pub fn token(&self) -> &str { + &self.token + } + + /// True when the store retained the original and retrieval may be advertised. + pub fn retained(&self) -> bool { + self.retained + } +} + +/// Injectable CCR store. +pub trait CcrStore: Send + Sync { + fn put(&self, content: &str) -> CcrPutResult; + fn get(&self, token: &str) -> Option; + fn get_range(&self, token: &str, start: usize, end: usize, unit: RangeUnit) -> Option { + let original = self.get(token)?; + range_from_original(&original, start, end, unit) + } +} + +/// Compatibility wrapper over the process-global CCR store. +#[derive(Debug, Default, Clone, Copy)] +pub struct GlobalCcrStore; + +impl CcrStore for GlobalCcrStore { + fn put(&self, content: &str) -> CcrPutResult { + let (token, retained) = offload_checked(content); + CcrPutResult::new(token, retained) + } + + fn get(&self, token: &str) -> Option { + retrieve(token) + } + + fn get_range(&self, token: &str, start: usize, end: usize, unit: RangeUnit) -> Option { + retrieve_range(token, start, end, unit) + } +} + +/// Isolated in-memory CCR store for tests and host-managed adapters. +pub struct MemoryCcrStore { + inner: Mutex, + max_entries: usize, + max_bytes: usize, + ttl: Option, + disk_root: Option, +} + +impl Default for MemoryCcrStore { + fn default() -> Self { + Self::new(DEFAULT_MAX_ENTRIES, DEFAULT_MAX_BYTES) + } +} + +impl MemoryCcrStore { + pub fn new(max_entries: usize, max_bytes: usize) -> Self { + Self { + inner: Mutex::new(Inner::default()), + max_entries: max_entries.max(1), + max_bytes: max_bytes.max(1), + ttl: None, + disk_root: None, + } + } + + pub fn with_ttl(mut self, ttl: Option) -> Self { + self.ttl = ttl; + self + } + + pub fn with_disk_root(mut self, root: PathBuf) -> Self { + if std::fs::create_dir_all(&root).is_ok() { + self.disk_root = Some(root); + } + self + } + + pub fn stats(&self) -> (usize, usize) { + let inner = self.inner.lock().unwrap_or_else(|p| p.into_inner()); + (inner.map.len(), inner.total_bytes) + } +} + +impl CcrStore for MemoryCcrStore { + fn put(&self, content: &str) -> CcrPutResult { + let token = short_hash(content); + let mem_retained = self.inner.lock().unwrap_or_else(|p| p.into_inner()).insert( + token.clone(), + content.to_string(), + self.max_entries, + self.max_bytes, + ); + let mut disk_retained = false; + if let Some(root) = self.disk_root.as_ref() { + let path = root.join(&token); + if std::fs::write(path, content).is_ok() { + disk_retained = true; + } + } + CcrPutResult::new(token, mem_retained || disk_retained) + } + + fn get(&self, token: &str) -> Option { + if !is_valid_token(token) { + return None; + } + { + let mut inner = self.inner.lock().unwrap_or_else(|p| p.into_inner()); + if let Some(entry) = inner.map.get(token) { + if self.ttl.is_none_or(|ttl| entry.created.elapsed() < ttl) { + return Some(entry.content.clone()); + } + if let Some(entry) = inner.map.remove(token) { + inner.total_bytes = inner.total_bytes.saturating_sub(entry.content.len()); + } + } + } + let root = self.disk_root.as_ref()?; + let path = root.join(token); + if disk_entry_expired(&path, self.ttl) { + let _ = std::fs::remove_file(path); + return None; + } + std::fs::read_to_string(path).ok() + } +} + /// Tunable limits, settable once at startup from the `[tokenjuice]` config. struct Limits { max_entries: usize, @@ -269,14 +411,23 @@ pub enum RangeUnit { /// only when the original isn't available at all. pub fn retrieve_range(hash: &str, start: usize, end: usize, unit: RangeUnit) -> Option { let original = retrieve(hash)?; + range_from_original(&original, start, end, unit) +} + +fn range_from_original( + original: &str, + start: usize, + end: usize, + unit: RangeUnit, +) -> Option { if end <= start { return Some(String::new()); } match unit { RangeUnit::Bytes => { // Clamp to char boundaries so we never split a UTF-8 sequence. - let s = floor_char_boundary(&original, start.min(original.len())); - let e = floor_char_boundary(&original, end.min(original.len())); + let s = floor_char_boundary(original, start.min(original.len())); + let e = floor_char_boundary(original, end.min(original.len())); Some(original[s..e].to_string()) } RangeUnit::Lines => { @@ -435,6 +586,46 @@ mod tests { ); } + #[test] + fn memory_store_is_isolated_from_global_cache() { + let store = MemoryCcrStore::new(10, 10_000); + let original = "isolated memory store payload delta ".repeat(20); + let put = store.put(&original); + + assert!(put.retained()); + assert_eq!(store.get(put.token()).as_deref(), Some(original.as_str())); + assert_eq!(retrieve(put.token()), None, "global cache must not see it"); + } + + #[test] + fn memory_store_rejects_oversized_originals() { + let store = MemoryCcrStore::new(10, 16); + let original = "oversized isolated payload echo ".repeat(20); + let put = store.put(&original); + + assert!(!put.retained()); + assert_eq!(store.get(put.token()), None); + assert_eq!(store.stats(), (0, 0)); + } + + #[test] + fn memory_store_rejects_malformed_tokens_before_disk_lookup() { + let dir = std::env::temp_dir().join(format!( + "tj-memory-ccr-{}", + short_hash("memory-store-malformed-token") + )); + let _ = std::fs::remove_dir_all(&dir); + let store = MemoryCcrStore::new(10, 10_000).with_disk_root(dir.clone()); + + assert_eq!(store.get("../../state/config.toml"), None); + assert_eq!( + store.get_range("../../state/config.toml", 0, 1, RangeUnit::Lines), + None + ); + + let _ = std::fs::remove_dir_all(&dir); + } + #[test] fn disk_tier_survives_memory_miss() { let dir = std::env::temp_dir().join(format!("tj-ccr-{}", short_hash("disk-test-seed"))); diff --git a/src/compress.rs b/src/compress.rs index 4aeb1e1..82fee21 100644 --- a/src/compress.rs +++ b/src/compress.rs @@ -12,8 +12,11 @@ //! [`CompressInput`] with a derived command/argv and calls [`route`]. use crate::cache; +use crate::cache::CcrStore; use crate::compressors::{compressor_for, generic_compressor}; use crate::detect::detect_content_kind; +use crate::pipeline::{PipelineReport, PipelineSkipReason, estimate_bloat}; +use crate::policy::{ShellCompactionPolicy, ShellPolicyDecision, apply_shell_compaction_policy}; use crate::savings; use crate::tokens::estimate_tokens; use crate::types::{ @@ -31,6 +34,31 @@ pub async fn compress_content( hint: Option, opts: &CompressOptions, ) -> CompressedOutput { + compress_content_with_store_report(content, hint, opts, &cache::GlobalCcrStore) + .await + .0 +} + +/// Store-injected variant of [`compress_content`]. +pub async fn compress_content_with_store( + content: &str, + hint: Option, + opts: &CompressOptions, + store: &dyn CcrStore, +) -> CompressedOutput { + compress_content_with_store_report(content, hint, opts, store) + .await + .0 +} + +/// Store-injected variant of [`compress_content`] that also returns a redacted +/// pipeline report. +pub async fn compress_content_with_store_report( + content: &str, + hint: Option, + opts: &CompressOptions, + store: &dyn CcrStore, +) -> (CompressedOutput, PipelineReport) { let hint = hint.unwrap_or_default(); let input = CompressInput { content, @@ -41,22 +69,82 @@ pub async fn compress_content( argv: None, original_bytes: content.len(), }; - route(input, opts).await + route_with_store_report(input, opts, store).await } /// Core router: detect (unless the input already carries a resolved kind via the /// hint's explicit override), pick the compressor honouring config gates, run /// it, and apply CCR offload + footer. -pub async fn route(mut input: CompressInput<'_>, opts: &CompressOptions) -> CompressedOutput { +pub async fn route(input: CompressInput<'_>, opts: &CompressOptions) -> CompressedOutput { + route_with_store_report(input, opts, &cache::GlobalCcrStore) + .await + .0 +} + +/// Store-injected variant of [`route`]. +pub async fn route_with_store( + input: CompressInput<'_>, + opts: &CompressOptions, + store: &dyn CcrStore, +) -> CompressedOutput { + route_with_store_report(input, opts, store).await.0 +} + +/// Store-injected variant of [`route`] that also returns a redacted pipeline +/// report. +pub async fn route_with_store_report( + mut input: CompressInput<'_>, + opts: &CompressOptions, + store: &dyn CcrStore, +) -> (CompressedOutput, PipelineReport) { let content = input.content; let original_bytes = content.len(); - if !opts.router_enabled || original_bytes < opts.min_bytes_to_compress { + if !opts.router_enabled { + let kind = detect_content_kind(content, input.hint); + let res = CompressedOutput::passthrough(content.to_string(), kind); + let report = + PipelineReport::passthrough(kind, original_bytes, PipelineSkipReason::RouterDisabled) + .with_bloat_estimate(estimate_bloat(content, kind)); + return (res, report); + } + + if original_bytes < opts.min_bytes_to_compress { let kind = detect_content_kind(content, input.hint); - return CompressedOutput::passthrough(content.to_string(), kind); + let res = CompressedOutput::passthrough(content.to_string(), kind); + let report = + PipelineReport::passthrough(kind, original_bytes, PipelineSkipReason::BelowMinBytes) + .with_bloat_estimate(estimate_bloat(content, kind)); + return (res, report); } let kind = detect_content_kind(content, input.hint); + let bloat_estimate = estimate_bloat(content, kind); + let shell_policy_decision = apply_shell_compaction_policy( + &crate::types::ToolExecutionInput { + tool_name: input + .hint + .source_tool + .clone() + .unwrap_or_else(|| "shell".to_owned()), + command: input.command.clone(), + argv: input.argv.clone(), + stdout: Some(content.to_owned()), + exit_code: input.exit_code, + ..Default::default() + }, + ShellCompactionPolicy::AllowSafeInventory, + ); + if !matches!(shell_policy_decision, ShellPolicyDecision::Compact) { + let res = CompressedOutput::passthrough(content.to_string(), kind); + let report = PipelineReport::passthrough( + kind, + original_bytes, + PipelineSkipReason::Other(shell_policy_decision.as_str()), + ) + .with_bloat_estimate(bloat_estimate); + return (res, report); + } input.kind = kind; // Resolve which compressor to try, honouring per-kind config gates. @@ -82,10 +170,18 @@ pub async fn route(mut input: CompressInput<'_>, opts: &CompressOptions) -> Comp } let Some(out) = produced else { - return CompressedOutput::passthrough(content.to_string(), kind); + let res = CompressedOutput::passthrough(content.to_string(), kind); + let report = + PipelineReport::passthrough(kind, original_bytes, PipelineSkipReason::NoCompressor) + .with_bloat_estimate(bloat_estimate); + return (res, report); }; if out.text.len() >= original_bytes { - return CompressedOutput::passthrough(content.to_string(), kind); + let res = CompressedOutput::passthrough(content.to_string(), kind); + let report = + PipelineReport::passthrough(kind, original_bytes, PipelineSkipReason::NoSavings) + .with_bloat_estimate(bloat_estimate); + return (res, report); } // CCR threshold: only offload (and therefore only allow *lossy* compaction) @@ -95,33 +191,62 @@ pub async fn route(mut input: CompressInput<'_>, opts: &CompressOptions) -> Comp let original_tokens = estimate_tokens(content); let ccr_for_call = opts.ccr_enabled && original_tokens as usize >= opts.ccr_min_tokens; if out.lossy && !ccr_for_call { - return CompressedOutput::passthrough(content.to_string(), kind); + let res = CompressedOutput::passthrough(content.to_string(), kind); + let report = + PipelineReport::passthrough(kind, original_bytes, PipelineSkipReason::CcrDisabled) + .with_bloat_estimate(bloat_estimate); + return (res, report); } - // Offload the original and append a recovery footer when CCR is in play. - let (text, ccr_token) = if ccr_for_call { - let (token, retained) = cache::offload_checked(content); - if !retained { + // Offload the original and expose the recovery footer separately. `text` + // below remains the compatibility output (body + footer), while hosts with + // their own caps can truncate `body` and reattach `recovery_footer`. + let (body, recovery_footer, ccr_token) = if ccr_for_call { + let put = store.put(content); + if !put.retained() { // The original is too large to keep in memory (over the byte cap) // and the disk tier isn't on, so it can't be recovered. A lossy view // would be irreversible — decline it. A lossless reformat is still // safe to return, just without a (dangling) recovery footer. if out.lossy { - return CompressedOutput::passthrough(content.to_string(), kind); + let res = CompressedOutput::passthrough(content.to_string(), kind); + let report = PipelineReport::passthrough( + kind, + original_bytes, + PipelineSkipReason::CcrNotRetained, + ) + .with_bloat_estimate(bloat_estimate); + return (res, report); } - (out.text, None) + (out.text, None, None) } else { + let token = put.token().to_string(); let footer = cache::recovery_footer(&token, original_bytes, out.lossy); - let mut text = out.text; + let mut text = out.text.clone(); text.push_str(&footer); // The footer adds bytes — if it tipped us over the original size, bail. if text.len() >= original_bytes { - return CompressedOutput::passthrough(content.to_string(), kind); + let res = CompressedOutput::passthrough(content.to_string(), kind); + let report = PipelineReport::passthrough( + kind, + original_bytes, + PipelineSkipReason::FooterWouldGrow, + ) + .with_bloat_estimate(bloat_estimate); + return (res, report); } - (text, Some(token)) + (out.text, Some(footer), Some(token)) } } else { - (out.text, None) + (out.text, None, None) + }; + + let text = if let Some(footer) = recovery_footer.as_deref() { + let mut text = body.clone(); + text.push_str(footer); + text + } else { + body.clone() }; let compacted_bytes = text.len(); @@ -141,8 +266,10 @@ pub async fn route(mut input: CompressInput<'_>, opts: &CompressOptions) -> Comp // result is being compressed for). savings::record(kind, out.kind, original_tokens, compacted_tokens); - CompressedOutput { + let res = CompressedOutput { text, + body, + recovery_footer, content_kind: kind, compressor: out.kind, lossy: out.lossy, @@ -150,7 +277,17 @@ pub async fn route(mut input: CompressInput<'_>, opts: &CompressOptions) -> Comp ccr_token, original_bytes, compacted_bytes, - } + }; + let report = PipelineReport::applied( + kind, + original_bytes, + compacted_bytes, + out.kind, + out.lossy, + res.ccr_token.clone(), + ) + .with_bloat_estimate(bloat_estimate); + (res, report) } /// Build a [`CompressorKind`] label-free passthrough quickly (used by callers @@ -194,6 +331,146 @@ mod tests { ); } + #[tokio::test] + async fn exposes_body_and_recovery_footer_separately() { + let mut rows = Vec::new(); + for i in 0..120 { + rows.push(format!( + r#"{{"id":{i},"name":"isolated_account_{i}","email":"iso{i}@ex.com","tier":"platinum"}}"# + )); + } + let original = format!("[{}]", rows.join(",")); + let res = compress_content(&original, None, &opts()).await; + + assert!(res.applied); + let footer = res + .recovery_footer + .as_deref() + .expect("offloaded output exposes footer"); + assert!(!res.body.contains("⟦tj:"), "body must not contain footer"); + assert!(footer.contains("⟦tj:"), "footer carries marker: {footer}"); + assert_eq!(res.text, format!("{}{}", res.body, footer)); + assert_eq!(res.compacted_bytes, res.text.len()); + } + + #[tokio::test] + async fn store_injected_route_uses_isolated_ccr_store() { + let store = cache::MemoryCcrStore::new(10, 1_000_000); + let mut rows = Vec::new(); + for i in 0..120 { + rows.push(format!( + r#"{{"id":{i},"name":"store_only_account_{i}","email":"store{i}@ex.com","tier":"silver"}}"# + )); + } + let original = format!("[{}]", rows.join(",")); + let res = compress_content_with_store(&original, None, &opts(), &store).await; + + assert!(res.applied); + let token = res.ccr_token.as_deref().expect("offloaded"); + assert_eq!(store.get(token).as_deref(), Some(original.as_str())); + assert_eq!(cache::retrieve(token), None, "global cache must not see it"); + } + + #[tokio::test] + async fn report_records_applied_step_and_ccr_token() { + let store = cache::MemoryCcrStore::new(10, 1_000_000); + let mut rows = Vec::new(); + for i in 0..120 { + rows.push(format!( + r#"{{"id":{i},"name":"report_account_{i}","email":"report{i}@ex.com","tier":"bronze"}}"# + )); + } + let original = format!("[{}]", rows.join(",")); + let (res, report) = + compress_content_with_store_report(&original, None, &opts(), &store).await; + + assert!(res.applied); + assert_eq!(report.content_kind, ContentKind::Json); + assert_eq!(report.original_bytes, original.len()); + assert_eq!(report.compacted_bytes, res.text.len()); + assert_eq!(report.applied_steps.len(), 1); + assert_eq!( + report.applied_steps[0].compressor, + Some(CompressorKind::SmartCrusher) + ); + assert_eq!( + report + .bloat_estimate + .map(|estimate| estimate.reason.as_str()), + Some("json_rows") + ); + assert_eq!(report.ccr_tokens, vec![res.ccr_token.unwrap()]); + assert_eq!(report.skip_reason, None); + } + + #[tokio::test] + async fn report_records_redacted_skip_reason() { + let mut o = opts(); + o.router_enabled = false; + let (res, report) = compress_content_with_store_report( + "sensitive raw payload", + None, + &o, + &cache::GlobalCcrStore, + ) + .await; + + assert!(!res.applied); + assert_eq!(report.skip_reason, Some(PipelineSkipReason::RouterDisabled)); + assert!(report.bloat_estimate.is_some()); + assert!(report.applied_steps.is_empty()); + assert!(report.ccr_tokens.is_empty()); + } + + #[tokio::test] + async fn shell_policy_keeps_exact_file_read_raw_even_with_extension_hint() { + let content = (0..120) + .map(|i| format!("fn generated_{i}() {{ println!(\"{i}\"); }}")) + .collect::>() + .join("\n"); + let hint = ContentHint { + source_tool: Some("shell".to_owned()), + extension: Some("rs".to_owned()), + ..Default::default() + }; + let input = CompressInput { + content: &content, + kind: ContentKind::PlainText, + hint: &hint, + exit_code: None, + command: Some("cat src/lib.rs".to_owned()), + argv: None, + original_bytes: content.len(), + }; + + let store = cache::MemoryCcrStore::new(10, 1_000_000); + let (res, report) = route_with_store_report(input, &opts(), &store).await; + + assert!(!res.applied); + assert_eq!(res.text, content); + assert_eq!( + report.skip_reason, + Some(PipelineSkipReason::Other("skip_file_content")) + ); + } + + #[tokio::test] + async fn lossy_output_declines_when_injected_store_cannot_retain() { + let store = cache::MemoryCcrStore::new(10, 16); + let mut rows = Vec::new(); + for i in 0..120 { + rows.push(format!( + r#"{{"id":{i},"name":"account_{i}","email":"a{i}@ex.com","tier":"gold"}}"# + )); + } + let original = format!("[{}]", rows.join(",")); + let res = compress_content_with_store(&original, None, &opts(), &store).await; + + assert!(!res.applied); + assert_eq!(res.text, original); + assert!(res.ccr_token.is_none()); + } + #[tokio::test] async fn small_input_passes_through() { let res = compress_content("tiny", None, &opts()).await; diff --git a/src/conversation/boundary.rs b/src/conversation/boundary.rs new file mode 100644 index 0000000..acd41e2 --- /dev/null +++ b/src/conversation/boundary.rs @@ -0,0 +1,186 @@ +use std::collections::HashSet; + +use crate::conversation::{ConversationMessage, ToolCall, ToolResultMessage}; + +/// Move a retained-tail start backward when needed so retained tool results +/// keep their parent assistant tool-call message. +pub fn align_tail_start_for_tool_boundaries( + messages: &[ConversationMessage], + tail_start: usize, +) -> usize { + let mut start = tail_start.min(messages.len()); + loop { + let retained_calls = call_ids_in_range(messages, start, messages.len()); + let missing_result_parent = messages[start..].iter().find_map(|message| match message { + ConversationMessage::ToolResults(results) => results + .iter() + .find(|result| !retained_calls.contains(&result.tool_call_id)) + .and_then(|result| find_parent_call_index(messages, start, &result.tool_call_id)), + _ => None, + }); + + let Some(parent_idx) = missing_result_parent else { + return start; + }; + if parent_idx >= start { + return start; + } + start = parent_idx; + } +} + +/// Remove provider-invalid orphan tool messages from a retained conversation. +pub fn sanitize_orphan_tool_messages(messages: &[ConversationMessage]) -> Vec { + let all_results = result_ids_in_range(messages, 0, messages.len()); + let all_calls = call_ids_in_range(messages, 0, messages.len()); + + messages + .iter() + .filter_map(|message| match message { + ConversationMessage::AssistantToolCalls { + text, + tool_calls, + metadata, + } => { + let kept: Vec = tool_calls + .iter() + .filter(|call| all_results.contains(&call.id)) + .cloned() + .collect(); + if kept.is_empty() && text.as_deref().unwrap_or("").is_empty() { + None + } else { + Some(ConversationMessage::AssistantToolCalls { + text: text.clone(), + tool_calls: kept, + metadata: metadata.clone(), + }) + } + } + ConversationMessage::ToolResults(results) => { + let kept: Vec = results + .iter() + .filter(|result| all_calls.contains(&result.tool_call_id)) + .cloned() + .collect(); + (!kept.is_empty()).then_some(ConversationMessage::ToolResults(kept)) + } + ConversationMessage::Chat(_) => Some(message.clone()), + }) + .collect() +} + +/// Latest user message that is not an internal compaction summary. +pub fn latest_real_user_index(messages: &[ConversationMessage]) -> Option { + messages.iter().rposition(|message| match message { + ConversationMessage::Chat(chat) => { + chat.role == "user" && !chat.is_compaction_summary && !chat.hidden + } + _ => false, + }) +} + +/// Latest visible assistant reply that is not an internal compaction summary. +pub fn latest_visible_assistant_index(messages: &[ConversationMessage]) -> Option { + messages.iter().rposition(|message| match message { + ConversationMessage::Chat(chat) => { + chat.role == "assistant" + && !chat.content.trim().is_empty() + && !chat.is_compaction_summary + && !chat.hidden + } + ConversationMessage::AssistantToolCalls { text, .. } => { + text.as_deref().is_some_and(|text| !text.trim().is_empty()) + } + _ => false, + }) +} + +fn call_ids_in_range( + messages: &[ConversationMessage], + start: usize, + end: usize, +) -> HashSet { + messages[start.min(messages.len())..end.min(messages.len())] + .iter() + .flat_map(|message| match message { + ConversationMessage::AssistantToolCalls { tool_calls, .. } => { + tool_calls.iter().map(|call| call.id.clone()).collect() + } + _ => Vec::new(), + }) + .collect() +} + +fn result_ids_in_range( + messages: &[ConversationMessage], + start: usize, + end: usize, +) -> HashSet { + messages[start.min(messages.len())..end.min(messages.len())] + .iter() + .flat_map(|message| match message { + ConversationMessage::ToolResults(results) => results + .iter() + .map(|result| result.tool_call_id.clone()) + .collect(), + _ => Vec::new(), + }) + .collect() +} + +fn find_parent_call_index( + messages: &[ConversationMessage], + before: usize, + call_id: &str, +) -> Option { + messages[..before.min(messages.len())] + .iter() + .rposition(|message| match message { + ConversationMessage::AssistantToolCalls { tool_calls, .. } => { + tool_calls.iter().any(|call| call.id == call_id) + } + _ => false, + }) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::conversation::{ChatMessage, ToolCall}; + + #[test] + fn retained_tool_result_aligns_to_parent_call() { + let messages = vec![ + ConversationMessage::user("ask"), + ConversationMessage::assistant_tool_calls(vec![ToolCall::new("a", "read_file", "{}")]), + ConversationMessage::tool_results(vec![ToolResultMessage::new("a", "file body")]), + ConversationMessage::assistant("done"), + ]; + + assert_eq!(align_tail_start_for_tool_boundaries(&messages, 2), 1); + } + + #[test] + fn orphan_tool_result_is_removed() { + let messages = vec![ + ConversationMessage::tool_results(vec![ToolResultMessage::new("missing", "body")]), + ConversationMessage::assistant("done"), + ]; + let sanitized = sanitize_orphan_tool_messages(&messages); + assert_eq!(sanitized, vec![ConversationMessage::assistant("done")]); + } + + #[test] + fn anchors_skip_internal_summary_messages() { + let messages = vec![ + ConversationMessage::Chat(ChatMessage::compaction_summary("user", "old ask")), + ConversationMessage::user("real ask"), + ConversationMessage::Chat(ChatMessage::compaction_summary("assistant", "summary")), + ConversationMessage::assistant("visible"), + ]; + + assert_eq!(latest_real_user_index(&messages), Some(1)); + assert_eq!(latest_visible_assistant_index(&messages), Some(3)); + } +} diff --git a/src/conversation/budget.rs b/src/conversation/budget.rs new file mode 100644 index 0000000..0248a58 --- /dev/null +++ b/src/conversation/budget.rs @@ -0,0 +1,293 @@ +use crate::conversation::ConversationMessage; +use crate::tokens::estimate_tokens; + +/// Request-window parameters for conversation compaction. +#[derive(Debug, Clone, Copy, PartialEq)] +pub struct ConversationBudget { + pub context_length: usize, + pub max_output_tokens: usize, + pub threshold_ratio: f32, + pub minimum_context_floor: usize, +} + +impl Default for ConversationBudget { + fn default() -> Self { + Self { + context_length: 128_000, + max_output_tokens: 16_384, + threshold_ratio: 0.85, + minimum_context_floor: 64_000, + } + } +} + +/// Non-sensitive details for a tail-budget decision. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct TailBudgetSelection { + pub tail_start: usize, + pub selected_tokens: usize, + pub message_count: usize, +} + +/// Options for preserving the beginning of a transcript. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct HeadProtection { + pub protect_system_prompt: bool, + pub protect_first_n_messages: usize, + pub decay_head_after_first_compaction: bool, + pub compaction_count: usize, +} + +impl Default for HeadProtection { + fn default() -> Self { + Self { + protect_system_prompt: true, + protect_first_n_messages: 0, + decay_head_after_first_compaction: true, + compaction_count: 0, + } + } +} + +/// Input window after reserving output tokens. +pub fn effective_input_window(context_length: usize, max_output_tokens: usize) -> usize { + context_length.saturating_sub(max_output_tokens) +} + +/// Threshold at which conversation compaction should trigger. +pub fn threshold_tokens(budget: ConversationBudget) -> usize { + let effective = effective_input_window(budget.context_length, budget.max_output_tokens); + if effective == 0 { + return 0; + } + + let ratio = budget.threshold_ratio.clamp(0.01, 0.99); + let ratio_threshold = ((effective as f32) * ratio).floor() as usize; + let threshold = ratio_threshold.max(budget.minimum_context_floor); + + if threshold >= effective { + ((effective as f32) * 0.85).floor().max(1.0) as usize + } else { + threshold + } +} + +/// Return the exclusive end index of the protected head. +pub fn protected_head_end(messages: &[ConversationMessage], options: HeadProtection) -> usize { + let system_end = if options.protect_system_prompt { + messages + .iter() + .take_while(|message| match message { + ConversationMessage::Chat(chat) => chat.role == "system", + _ => false, + }) + .count() + } else { + 0 + }; + + let keep_initial_head = + !options.decay_head_after_first_compaction || options.compaction_count == 0; + let first_n_end = if keep_initial_head { + options.protect_first_n_messages.min(messages.len()) + } else { + 0 + }; + + system_end.max(first_n_end) +} + +/// Rough token estimate for a provider-neutral message, including envelope +/// overhead and tool-call argument cost. +pub fn estimate_message_tokens(message: &ConversationMessage) -> usize { + const ROLE_OVERHEAD: usize = 4; + const TOOL_CALL_OVERHEAD: usize = 12; + const TOOL_RESULT_OVERHEAD: usize = 8; + + match message { + ConversationMessage::Chat(chat) => { + ROLE_OVERHEAD + + estimate_tokens(&chat.role) as usize + + estimate_tokens(&chat.content) as usize + } + ConversationMessage::AssistantToolCalls { + text, tool_calls, .. + } => { + let text_tokens = text.as_deref().map(estimate_tokens).unwrap_or(0) as usize; + let tool_tokens = tool_calls + .iter() + .map(|call| { + TOOL_CALL_OVERHEAD + + estimate_tokens(&call.id) as usize + + estimate_tokens(&call.name) as usize + + estimate_tokens(&call.arguments) as usize + }) + .sum::(); + ROLE_OVERHEAD + text_tokens + tool_tokens + } + ConversationMessage::ToolResults(results) => { + ROLE_OVERHEAD + + results + .iter() + .map(|result| { + TOOL_RESULT_OVERHEAD + + estimate_tokens(&result.tool_call_id) as usize + + estimate_tokens(&result.content) as usize + }) + .sum::() + } + } +} + +/// Select the start index of the recent tail to keep under a rough token +/// budget. Returns an index in `messages`. +pub fn select_tail_by_budget( + messages: &[ConversationMessage], + head_end: usize, + token_budget: usize, + min_tail_messages: usize, + soft_ceiling_ratio: f32, +) -> usize { + select_tail_by_budget_detail( + messages, + head_end, + token_budget, + min_tail_messages, + soft_ceiling_ratio, + ) + .tail_start +} + +pub fn select_tail_by_budget_detail( + messages: &[ConversationMessage], + head_end: usize, + token_budget: usize, + min_tail_messages: usize, + soft_ceiling_ratio: f32, +) -> TailBudgetSelection { + let head_end = head_end.min(messages.len()); + if head_end >= messages.len() { + return TailBudgetSelection { + tail_start: messages.len(), + selected_tokens: 0, + message_count: 0, + }; + } + + let region = &messages[head_end..]; + let region_tokens = region.iter().map(estimate_message_tokens).sum::(); + let soft_ceiling = ((token_budget as f32) * soft_ceiling_ratio.max(1.0)).ceil() as usize; + if region_tokens <= soft_ceiling { + return TailBudgetSelection { + tail_start: head_end, + selected_tokens: region_tokens, + message_count: region.len(), + }; + } + + let floor = min_tail_messages.min(messages.len() - head_end); + let mut selected_tokens = 0usize; + let mut selected_count = 0usize; + let mut tail_start = messages.len(); + + for idx in (head_end..messages.len()).rev() { + let msg_tokens = estimate_message_tokens(&messages[idx]); + let must_keep_for_floor = selected_count < floor; + let within_budget = selected_tokens.saturating_add(msg_tokens) <= token_budget; + let allow_single_overshoot = + selected_count == 0 && msg_tokens <= soft_ceiling.max(token_budget); + + if must_keep_for_floor || within_budget || allow_single_overshoot { + selected_tokens = selected_tokens.saturating_add(msg_tokens); + selected_count += 1; + tail_start = idx; + } else { + break; + } + } + + TailBudgetSelection { + tail_start, + selected_tokens, + message_count: selected_count, + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::conversation::{ChatMessage, ToolResultMessage}; + + #[test] + fn threshold_stays_below_small_effective_window() { + let budget = ConversationBudget { + context_length: 64_000, + max_output_tokens: 8_000, + threshold_ratio: 0.9, + minimum_context_floor: 64_000, + }; + let effective = effective_input_window(budget.context_length, budget.max_output_tokens); + assert!(threshold_tokens(budget) < effective); + } + + #[test] + fn output_reservation_reduces_input_budget() { + assert_eq!(effective_input_window(128_000, 16_000), 112_000); + } + + #[test] + fn huge_old_tool_result_does_not_force_preserving_recent_history() { + let messages = vec![ + ConversationMessage::Chat(ChatMessage::system("sys")), + ConversationMessage::tool_results(vec![ToolResultMessage::new( + "old", + "x".repeat(20_000), + )]), + ConversationMessage::user("latest ask"), + ConversationMessage::assistant("latest answer"), + ]; + + let start = select_tail_by_budget(&messages, 1, 20, 2, 1.5); + assert_eq!(start, 2); + } + + #[test] + fn short_transcript_selects_whole_region() { + let messages = vec![ + ConversationMessage::system("sys"), + ConversationMessage::user("hi"), + ConversationMessage::assistant("hello"), + ]; + assert_eq!(select_tail_by_budget(&messages, 1, 100, 2, 1.25), 1); + } + + #[test] + fn protected_head_decays_after_first_compaction_but_keeps_system() { + let messages = vec![ + ConversationMessage::system("sys"), + ConversationMessage::user("opening task"), + ConversationMessage::assistant("ack"), + ConversationMessage::user("latest"), + ]; + + let first = protected_head_end( + &messages, + HeadProtection { + protect_first_n_messages: 3, + compaction_count: 0, + ..Default::default() + }, + ); + let repeated = protected_head_end( + &messages, + HeadProtection { + protect_first_n_messages: 3, + compaction_count: 1, + ..Default::default() + }, + ); + + assert_eq!(first, 3); + assert_eq!(repeated, 1); + } +} diff --git a/src/conversation/mod.rs b/src/conversation/mod.rs new file mode 100644 index 0000000..52ae657 --- /dev/null +++ b/src/conversation/mod.rs @@ -0,0 +1,23 @@ +//! Provider-neutral conversation compaction primitives. +//! +//! This module is intentionally pure library code: it does not know about +//! OpenHuman runtime types, provider payloads, filesystems, or model calls. + +mod boundary; +mod budget; +mod tool_digest; +mod types; + +pub use boundary::{ + align_tail_start_for_tool_boundaries, latest_real_user_index, latest_visible_assistant_index, + sanitize_orphan_tool_messages, +}; +pub use budget::{ + ConversationBudget, HeadProtection, TailBudgetSelection, effective_input_window, + estimate_message_tokens, protected_head_end, select_tail_by_budget, threshold_tokens, +}; +pub use tool_digest::{ + ToolDigestEntry, ToolDigestOptions, ToolDigestReport, digest_old_tool_results, + redact_sensitive_json, shrink_json_string_leaves, +}; +pub use types::{ChatMessage, ConversationMessage, ToolCall, ToolResultMessage}; diff --git a/src/conversation/tool_digest.rs b/src/conversation/tool_digest.rs new file mode 100644 index 0000000..2f17b0d --- /dev/null +++ b/src/conversation/tool_digest.rs @@ -0,0 +1,512 @@ +use std::collections::HashMap; + +use serde_json::{Map, Value}; +use sha2::{Digest, Sha256}; + +use crate::conversation::{ConversationMessage, ToolCall, ToolResultMessage}; + +/// Options for deterministic old tool-result digesting. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct ToolDigestOptions { + pub max_full_result_chars: usize, + pub keep_recent_tool_results: usize, + pub max_argument_string_chars: usize, +} + +impl Default for ToolDigestOptions { + fn default() -> Self { + Self { + max_full_result_chars: 2_000, + keep_recent_tool_results: 4, + max_argument_string_chars: 160, + } + } +} + +/// Redacted report entry for a replaced tool result. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ToolDigestEntry { + pub tool_call_id: String, + pub tool_name: Option, + pub hash: String, + pub original_bytes: usize, + pub replacement_bytes: usize, + pub duplicate_of: Option, +} + +/// Redacted report for a digest pass. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct ToolDigestReport { + pub entries: Vec, +} + +/// Shrink only string leaves in a JSON value, preserving valid JSON structure. +pub fn shrink_json_string_leaves(value: &Value, max_string_chars: usize) -> Value { + match value { + Value::String(s) => Value::String(shrink_string(s, max_string_chars)), + Value::Array(items) => Value::Array( + items + .iter() + .map(|item| shrink_json_string_leaves(item, max_string_chars)) + .collect(), + ), + Value::Object(map) => Value::Object( + map.iter() + .map(|(key, value)| { + ( + key.clone(), + shrink_json_string_leaves(value, max_string_chars), + ) + }) + .collect(), + ), + _ => value.clone(), + } +} + +/// Redact JSON values under sensitive-looking keys, then shrink string leaves. +pub fn redact_sensitive_json(value: &Value, max_string_chars: usize) -> Value { + redact_sensitive_json_inner(value, max_string_chars, None) +} + +/// Replace older duplicate/large tool results with deterministic digests. +pub fn digest_old_tool_results( + messages: &[ConversationMessage], + options: ToolDigestOptions, +) -> (Vec, ToolDigestReport) { + let calls = collect_calls(messages, options.max_argument_string_chars); + let occurrences = collect_result_occurrences(messages); + let total_results = occurrences.len(); + let mut newest_by_hash: HashMap = HashMap::new(); + for (serial, _, _, result) in &occurrences { + newest_by_hash.insert(stable_hash(&result.content), *serial); + } + + let mut report = ToolDigestReport::default(); + let mut serial_cursor = 0usize; + let mut out = Vec::with_capacity(messages.len()); + + for message in messages { + match message { + ConversationMessage::ToolResults(results) => { + let mut rendered = Vec::with_capacity(results.len()); + for result in results { + let serial = serial_cursor; + serial_cursor += 1; + let protected_recent = + total_results.saturating_sub(serial) <= options.keep_recent_tool_results; + let hash = stable_hash(&result.content); + let duplicate_of = newest_by_hash + .get(&hash) + .copied() + .filter(|newest| *newest != serial) + .and_then(|newest| { + occurrences + .iter() + .find(|(candidate_serial, _, _, _)| *candidate_serial == newest) + .map(|(_, _, _, newest_result)| newest_result.tool_call_id.clone()) + }); + + let should_digest = !protected_recent + && (duplicate_of.is_some() + || result.content.chars().count() > options.max_full_result_chars); + + if should_digest { + let call = calls.get(&result.tool_call_id); + let replacement = + render_digest(result, call, &hash, duplicate_of.as_deref()); + report.entries.push(ToolDigestEntry { + tool_call_id: result.tool_call_id.clone(), + tool_name: call.map(|call| call.name.clone()), + hash: hash.clone(), + original_bytes: result.content.len(), + replacement_bytes: replacement.len(), + duplicate_of, + }); + rendered.push(ToolResultMessage::new( + result.tool_call_id.clone(), + replacement, + )); + } else { + rendered.push(result.clone()); + } + } + out.push(ConversationMessage::ToolResults(rendered)); + } + ConversationMessage::AssistantToolCalls { + text, + tool_calls, + metadata, + } => out.push(ConversationMessage::AssistantToolCalls { + text: text.clone(), + tool_calls: tool_calls + .iter() + .map(|call| sanitize_tool_call(call, options.max_argument_string_chars)) + .collect(), + metadata: metadata.clone(), + }), + ConversationMessage::Chat(_) => out.push(message.clone()), + } + } + + (out, report) +} + +#[derive(Debug, Clone)] +struct SanitizedToolCall { + name: String, + arguments: String, +} + +fn collect_calls( + messages: &[ConversationMessage], + max_argument_string_chars: usize, +) -> HashMap { + let mut out = HashMap::new(); + for message in messages { + if let ConversationMessage::AssistantToolCalls { tool_calls, .. } = message { + for call in tool_calls { + let sanitized = sanitize_tool_call(call, max_argument_string_chars); + out.insert( + call.id.clone(), + SanitizedToolCall { + name: sanitized.name, + arguments: sanitized.arguments, + }, + ); + } + } + } + out +} + +fn sanitize_tool_call(call: &ToolCall, max_argument_string_chars: usize) -> ToolCall { + let arguments = match serde_json::from_str::(&call.arguments) { + Ok(value) => redact_sensitive_json(&value, max_argument_string_chars).to_string(), + Err(_) => redact_sensitive_text(&call.arguments, max_argument_string_chars), + }; + ToolCall { + id: call.id.clone(), + name: call.name.clone(), + arguments, + } +} + +fn collect_result_occurrences( + messages: &[ConversationMessage], +) -> Vec<(usize, usize, usize, ToolResultMessage)> { + let mut out = Vec::new(); + let mut serial = 0usize; + for (message_idx, message) in messages.iter().enumerate() { + if let ConversationMessage::ToolResults(results) = message { + for (result_idx, result) in results.iter().enumerate() { + out.push((serial, message_idx, result_idx, result.clone())); + serial += 1; + } + } + } + out +} + +fn render_digest( + result: &ToolResultMessage, + call: Option<&SanitizedToolCall>, + hash: &str, + duplicate_of: Option<&str>, +) -> String { + let bytes = result.content.len(); + let chars = result.content.chars().count(); + let lines = result.content.lines().count(); + let tool_name = call + .map(|call| call.name.as_str()) + .unwrap_or("unknown_tool"); + + if let Some(newer_id) = duplicate_of { + return format!( + "[tokenjuice tool-result duplicate: call_id={} newer_call_id={} tool={} bytes={} chars={} lines={} hash={}]", + result.tool_call_id, newer_id, tool_name, bytes, chars, lines, hash + ); + } + + let detail = call + .map(|call| render_tool_detail(tool_name, &call.arguments)) + .unwrap_or_default(); + if detail.is_empty() { + format!( + "[tokenjuice tool-result digest: call_id={} tool={} bytes={} chars={} lines={} hash={}]", + result.tool_call_id, tool_name, bytes, chars, lines, hash + ) + } else { + format!( + "[tokenjuice tool-result digest: call_id={} tool={} {} bytes={} chars={} lines={} hash={}]", + result.tool_call_id, tool_name, detail, bytes, chars, lines, hash + ) + } +} + +fn render_tool_detail(tool_name: &str, arguments: &str) -> String { + let Ok(value) = serde_json::from_str::(arguments) else { + return String::new(); + }; + let Some(obj) = value.as_object() else { + return String::new(); + }; + + match tool_name { + "shell" | "bash" | "exec" => first_string(obj, &["command", "cmd"]) + .map(|command| format!("command={}", compact_label(command))) + .unwrap_or_default(), + "read_file" | "file_read" | "fs_read" => first_string(obj, &["path", "file", "uri"]) + .map(|path| format!("path={}", compact_label(path))) + .unwrap_or_default(), + "search" | "grep" | "ripgrep" => first_string(obj, &["pattern", "query"]) + .map(|pattern| format!("pattern={}", compact_label(pattern))) + .unwrap_or_default(), + "web_fetch" | "fetch" | "browser_navigate" => first_string(obj, &["url", "uri"]) + .map(|url| format!("url={}", compact_label(url))) + .unwrap_or_default(), + _ => String::new(), + } +} + +fn first_string<'a>(obj: &'a Map, keys: &[&str]) -> Option<&'a str> { + keys.iter() + .find_map(|key| obj.get(*key).and_then(Value::as_str)) +} + +fn compact_label(value: &str) -> String { + shrink_string(&redact_sensitive_text(value, 80), 80).replace('\n', " ") +} + +fn redact_sensitive_json_inner(value: &Value, max_string_chars: usize, key: Option<&str>) -> Value { + if key.is_some_and(is_sensitive_key) { + return Value::String("[redacted]".to_owned()); + } + + match value { + Value::String(s) => Value::String(shrink_string( + &redact_sensitive_text(s, max_string_chars), + max_string_chars, + )), + Value::Array(items) => Value::Array( + items + .iter() + .map(|item| redact_sensitive_json_inner(item, max_string_chars, None)) + .collect(), + ), + Value::Object(map) => Value::Object( + map.iter() + .map(|(key, value)| { + ( + key.clone(), + redact_sensitive_json_inner(value, max_string_chars, Some(key)), + ) + }) + .collect(), + ), + _ => value.clone(), + } +} + +fn is_sensitive_key(key: &str) -> bool { + let normalized = key + .chars() + .filter(|ch| ch.is_ascii_alphanumeric()) + .collect::() + .to_ascii_lowercase(); + matches!( + normalized.as_str(), + "password" + | "passwd" + | "pwd" + | "secret" + | "token" + | "apikey" + | "apiKey" + | "authorization" + | "cookie" + | "setcookie" + ) || normalized.ends_with("token") + || normalized.ends_with("secret") + || normalized.ends_with("password") +} + +fn shrink_string(s: &str, max_chars: usize) -> String { + if max_chars == 0 { + return String::new(); + } + let char_count = s.chars().count(); + if char_count <= max_chars { + return s.to_owned(); + } + let keep = max_chars.saturating_sub(1); + let prefix = s.chars().take(keep).collect::(); + format!("{prefix}...[truncated {} chars]", char_count - keep) +} + +fn redact_sensitive_text(text: &str, max_chars: usize) -> String { + let mut out = text.to_owned(); + for marker in [ + "Bearer ", + "bearer ", + "token=", + "password=", + "secret=", + "api_key=", + ] { + out = redact_after_marker(&out, marker); + } + shrink_string(&out, max_chars) +} + +fn redact_after_marker(text: &str, marker: &str) -> String { + let mut remaining = text; + let mut out = String::with_capacity(text.len()); + while let Some(idx) = remaining.find(marker) { + let (before, after_before) = remaining.split_at(idx); + out.push_str(before); + out.push_str(marker); + out.push_str("[redacted]"); + let after = &after_before[marker.len()..]; + let end = after + .find(|ch: char| ch.is_whitespace() || matches!(ch, '\'' | '"' | '&' | ';')) + .unwrap_or(after.len()); + remaining = &after[end..]; + } + out.push_str(remaining); + out +} + +fn stable_hash(content: &str) -> String { + let digest = Sha256::digest(content.as_bytes()); + hex::encode(&digest[..8]) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::conversation::ToolCall; + + #[test] + fn shrink_json_string_leaves_preserves_valid_json_shape() { + let value = serde_json::json!({ + "path": "src/lib.rs", + "body": "abcdefghijklmnopqrstuvwxyz", + "nested": ["short", {"long": "0123456789abcdef"}] + }); + let shrunk = shrink_json_string_leaves(&value, 8); + assert_eq!(shrunk["path"], "src/lib...[truncated 3 chars]"); + assert!( + shrunk["nested"][1]["long"] + .as_str() + .unwrap() + .contains("[truncated") + ); + serde_json::to_string(&shrunk).expect("valid json"); + } + + #[test] + fn duplicate_read_file_outputs_keep_newest_full_copy() { + let body = "same file body\n".repeat(100); + let messages = vec![ + ConversationMessage::assistant_tool_calls(vec![ToolCall::new( + "old", + "read_file", + r#"{"path":"src/lib.rs"}"#, + )]), + ConversationMessage::tool_results(vec![ToolResultMessage::new("old", body.clone())]), + ConversationMessage::assistant_tool_calls(vec![ToolCall::new( + "new", + "read_file", + r#"{"path":"src/lib.rs"}"#, + )]), + ConversationMessage::tool_results(vec![ToolResultMessage::new("new", body.clone())]), + ]; + + let (out, report) = digest_old_tool_results( + &messages, + ToolDigestOptions { + max_full_result_chars: 10, + keep_recent_tool_results: 1, + max_argument_string_chars: 80, + }, + ); + + let ConversationMessage::ToolResults(old_results) = &out[1] else { + panic!("expected tool results"); + }; + let ConversationMessage::ToolResults(new_results) = &out[3] else { + panic!("expected tool results"); + }; + assert!(old_results[0].content.contains("duplicate")); + assert_eq!(new_results[0].content, body); + assert_eq!(report.entries[0].duplicate_of.as_deref(), Some("new")); + } + + #[test] + fn old_terminal_output_becomes_digest_with_command_exit_and_lines() { + let messages = vec![ + ConversationMessage::assistant_tool_calls(vec![ToolCall::new( + "cmd", + "shell", + r#"{"command":"cargo test"}"#, + )]), + ConversationMessage::tool_results(vec![ToolResultMessage::new( + "cmd", + "line\n".repeat(100), + )]), + ConversationMessage::user("next"), + ]; + + let (out, report) = digest_old_tool_results( + &messages, + ToolDigestOptions { + max_full_result_chars: 10, + keep_recent_tool_results: 0, + max_argument_string_chars: 80, + }, + ); + + let ConversationMessage::ToolResults(results) = &out[1] else { + panic!("expected tool results"); + }; + assert!(results[0].content.contains("tool=shell")); + assert!(results[0].content.contains("command=cargo test")); + assert!(results[0].content.contains("lines=100")); + assert_eq!(report.entries.len(), 1); + } + + #[test] + fn sensitive_values_are_redacted_before_digesting() { + let messages = vec![ + ConversationMessage::assistant_tool_calls(vec![ToolCall::new( + "cmd", + "shell", + r#"{"command":"curl -H 'Authorization: Bearer SECRET123'","token":"SECRET123"}"#, + )]), + ConversationMessage::tool_results(vec![ToolResultMessage::new( + "cmd", + "line\n".repeat(100), + )]), + ]; + + let (out, _) = digest_old_tool_results( + &messages, + ToolDigestOptions { + max_full_result_chars: 10, + keep_recent_tool_results: 0, + max_argument_string_chars: 120, + }, + ); + + let ConversationMessage::AssistantToolCalls { tool_calls, .. } = &out[0] else { + panic!("expected calls"); + }; + let ConversationMessage::ToolResults(results) = &out[1] else { + panic!("expected results"); + }; + assert!(!tool_calls[0].arguments.contains("SECRET123")); + assert!(!results[0].content.contains("SECRET123")); + assert!(tool_calls[0].arguments.contains("[redacted]")); + } +} diff --git a/src/conversation/types.rs b/src/conversation/types.rs new file mode 100644 index 0000000..1a81e5a --- /dev/null +++ b/src/conversation/types.rs @@ -0,0 +1,146 @@ +use serde::{Deserialize, Serialize}; +use serde_json::Value; + +/// A regular chat message in a provider-neutral conversation. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ChatMessage { + pub role: String, + pub content: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub id: Option, + /// Internal compaction summaries are not considered "real" user/assistant + /// anchors when selecting protected conversation turns. + #[serde(default)] + pub is_compaction_summary: bool, + /// UI-hidden assistant/system messages are skipped by visible-assistant + /// anchoring. + #[serde(default)] + pub hidden: bool, + /// Redacted metadata for host adapters. Core helpers never inspect raw host + /// runtime types. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub metadata: Option, +} + +impl ChatMessage { + pub fn system(content: impl Into) -> Self { + Self::new("system", content) + } + + pub fn user(content: impl Into) -> Self { + Self::new("user", content) + } + + pub fn assistant(content: impl Into) -> Self { + Self::new("assistant", content) + } + + pub fn tool(content: impl Into) -> Self { + Self::new("tool", content) + } + + pub fn compaction_summary(role: impl Into, content: impl Into) -> Self { + let mut msg = Self::new(role, content); + msg.is_compaction_summary = true; + msg + } + + fn new(role: impl Into, content: impl Into) -> Self { + Self { + role: role.into(), + content: content.into(), + id: None, + is_compaction_summary: false, + hidden: false, + metadata: None, + } + } +} + +/// A tool call requested by an assistant turn. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ToolCall { + pub id: String, + pub name: String, + /// Serialized JSON arguments when available. Non-JSON strings are preserved + /// by shrink/redaction helpers. + pub arguments: String, +} + +impl ToolCall { + pub fn new( + id: impl Into, + name: impl Into, + arguments: impl Into, + ) -> Self { + Self { + id: id.into(), + name: name.into(), + arguments: arguments.into(), + } + } +} + +/// A result returned for a tool call. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ToolResultMessage { + pub tool_call_id: String, + pub content: String, +} + +impl ToolResultMessage { + pub fn new(tool_call_id: impl Into, content: impl Into) -> Self { + Self { + tool_call_id: tool_call_id.into(), + content: content.into(), + } + } +} + +/// Provider-neutral conversation row. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "type", content = "data", rename_all = "camelCase")] +pub enum ConversationMessage { + Chat(ChatMessage), + AssistantToolCalls { + #[serde(default, skip_serializing_if = "Option::is_none")] + text: Option, + tool_calls: Vec, + #[serde(default, skip_serializing_if = "Option::is_none")] + metadata: Option, + }, + ToolResults(Vec), +} + +impl ConversationMessage { + pub fn chat(role: impl Into, content: impl Into) -> Self { + Self::Chat(ChatMessage::new(role, content)) + } + + pub fn system(content: impl Into) -> Self { + Self::Chat(ChatMessage::system(content)) + } + + pub fn user(content: impl Into) -> Self { + Self::Chat(ChatMessage::user(content)) + } + + pub fn assistant(content: impl Into) -> Self { + Self::Chat(ChatMessage::assistant(content)) + } + + pub fn assistant_tool_calls(tool_calls: Vec) -> Self { + Self::AssistantToolCalls { + text: None, + tool_calls, + metadata: None, + } + } + + pub fn tool_results(results: Vec) -> Self { + Self::ToolResults(results) + } +} diff --git a/src/lib.rs b/src/lib.rs index a396f2e..c80d08f 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -10,10 +10,13 @@ pub mod compress; pub mod compressor; pub mod compressors; pub mod config; +pub mod conversation; pub mod detect; mod error; pub mod ml; pub mod openhuman; +pub mod pipeline; +pub mod policy; pub mod reduce; pub mod rules; pub mod savings; @@ -23,14 +26,30 @@ pub mod tool_integration; pub mod types; mod util; -pub use compress::{compress_content, route}; +pub use compress::{ + compress_content, compress_content_with_store, compress_content_with_store_report, route, + route_with_store, route_with_store_report, +}; pub use compressor::{ CompressionInput, CompressionOutput, CompressionReport, Compressor, PassthroughCompressor, }; pub use compressors::{compressor_for, generic_compressor}; pub use config::CompressionConfig; +pub use conversation::{ + ChatMessage, ConversationBudget, ConversationMessage, HeadProtection, TailBudgetSelection, + ToolCall, ToolDigestEntry, ToolDigestOptions, ToolDigestReport, ToolResultMessage, + align_tail_start_for_tool_boundaries, digest_old_tool_results, effective_input_window, + estimate_message_tokens, latest_real_user_index, latest_visible_assistant_index, + protected_head_end, redact_sensitive_json, sanitize_orphan_tool_messages, + select_tail_by_budget, shrink_json_string_leaves, threshold_tokens, +}; pub use detect::detect_content_kind; pub use error::{TinyJuiceError, TinyJuiceResult}; +pub use pipeline::{ + OffloadOutput, OffloadTransform, PipelineInput, PipelineReport, PipelineSkipReason, + PipelineStep, ReformatTransform, TransformOutput, +}; +pub use policy::{ShellCompactionPolicy, ShellPolicyDecision, apply_shell_compaction_policy}; pub use reduce::reduce_execution_with_rules; pub use rules::{LoadRuleOptions, load_builtin_rules, load_rules}; pub use tool_integration::{ diff --git a/src/pipeline/estimate.rs b/src/pipeline/estimate.rs new file mode 100644 index 0000000..c680f49 --- /dev/null +++ b/src/pipeline/estimate.rs @@ -0,0 +1,248 @@ +use crate::types::ContentKind; + +/// Redacted estimate of how much non-essential bulk a payload appears to carry. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct BloatEstimate { + /// Heuristic score in the range 0..=100. + pub score: u8, + /// Dominant signal behind the score. Never contains raw content. + pub reason: BloatReason, +} + +/// Categorical bloat reason. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum BloatReason { + JsonRows, + DiffContext, + LogRepetition, + SearchFanout, + HtmlMarkup, + CodeBody, + TextRepetition, + LowSignal, +} + +impl BloatReason { + pub fn as_str(self) -> &'static str { + match self { + Self::JsonRows => "json_rows", + Self::DiffContext => "diff_context", + Self::LogRepetition => "log_repetition", + Self::SearchFanout => "search_fanout", + Self::HtmlMarkup => "html_markup", + Self::CodeBody => "code_body", + Self::TextRepetition => "text_repetition", + Self::LowSignal => "low_signal", + } + } +} + +/// Estimate bloat without retaining or reporting raw content. +pub fn estimate_bloat(content: &str, kind: ContentKind) -> BloatEstimate { + match kind { + ContentKind::Json => estimate_json_bloat(content), + ContentKind::Diff => estimate_diff_bloat(content), + ContentKind::Log => estimate_log_bloat(content), + ContentKind::Search => estimate_search_bloat(content), + ContentKind::Html => estimate_html_bloat(content), + ContentKind::Code => estimate_code_bloat(content), + ContentKind::PlainText => estimate_text_bloat(content), + } +} + +fn score_ratio(numerator: usize, denominator: usize) -> u8 { + if denominator == 0 { + return 0; + } + ((numerator.min(denominator) * 100) / denominator) as u8 +} + +fn estimate_json_bloat(content: &str) -> BloatEstimate { + let Ok(value) = serde_json::from_str::(content.trim()) else { + return low(); + }; + let serde_json::Value::Array(rows) = value else { + return low(); + }; + if rows.len() < 2 { + return low(); + } + let object_rows = rows.iter().filter(|row| row.is_object()).count(); + let score = (40 + score_ratio(object_rows.saturating_sub(2), rows.len()) / 2).min(95); + BloatEstimate { + score, + reason: BloatReason::JsonRows, + } +} + +fn estimate_diff_bloat(content: &str) -> BloatEstimate { + let mut context = 0usize; + let mut changed = 0usize; + for line in content.lines().take(2_000) { + if line.starts_with("@@") + || line.starts_with("diff --git") + || line.starts_with("+++") + || line.starts_with("---") + { + continue; + } + if line.starts_with('+') || line.starts_with('-') { + changed += 1; + } else if line.starts_with(' ') || !line.trim().is_empty() { + context += 1; + } + } + let total = context + changed; + if total == 0 { + return low(); + } + BloatEstimate { + score: score_ratio(context, total), + reason: BloatReason::DiffContext, + } +} + +fn estimate_log_bloat(content: &str) -> BloatEstimate { + let mut total = 0usize; + let mut repeated = 0usize; + let mut previous = ""; + for line in content.lines().take(2_000) { + let trimmed = line.trim(); + if trimmed.is_empty() { + continue; + } + total += 1; + if trimmed == previous { + repeated += 1; + } + previous = trimmed; + } + if total == 0 { + return low(); + } + BloatEstimate { + score: score_ratio(repeated + total.saturating_sub(80), total), + reason: BloatReason::LogRepetition, + } +} + +fn estimate_search_bloat(content: &str) -> BloatEstimate { + let mut total = 0usize; + let mut paths = std::collections::HashSet::new(); + for line in content.lines().take(2_000) { + let Some((path, _rest)) = line.split_once(':') else { + continue; + }; + if path.trim().is_empty() { + continue; + } + total += 1; + paths.insert(path); + } + if total == 0 { + return low(); + } + let fanout = paths.len(); + BloatEstimate { + score: (score_ratio(total.saturating_sub(20), total) / 2 + + score_ratio(fanout.saturating_sub(3), fanout.max(1)) / 2) + .min(95), + reason: BloatReason::SearchFanout, + } +} + +fn estimate_html_bloat(content: &str) -> BloatEstimate { + let tag_bytes = content + .bytes() + .filter(|byte| matches!(byte, b'<' | b'>')) + .count(); + BloatEstimate { + score: score_ratio(tag_bytes * 10, content.len().max(1)).min(95), + reason: BloatReason::HtmlMarkup, + } +} + +fn estimate_code_bloat(content: &str) -> BloatEstimate { + let mut total = 0usize; + let mut body_like = 0usize; + for line in content.lines().take(2_000) { + let trimmed = line.trim(); + if trimmed.is_empty() { + continue; + } + total += 1; + if trimmed == "{" || trimmed == "}" || trimmed.ends_with(';') || trimmed.contains(" = ") { + body_like += 1; + } + } + if total == 0 { + return low(); + } + BloatEstimate { + score: score_ratio(body_like, total), + reason: BloatReason::CodeBody, + } +} + +fn estimate_text_bloat(content: &str) -> BloatEstimate { + let mut total = 0usize; + let mut repeated = 0usize; + let mut seen = std::collections::HashSet::new(); + for line in content.lines().take(2_000) { + let trimmed = line.trim(); + if trimmed.is_empty() { + continue; + } + total += 1; + if !seen.insert(trimmed) { + repeated += 1; + } + } + if total == 0 || repeated == 0 { + return low(); + } + BloatEstimate { + score: score_ratio(repeated, total), + reason: BloatReason::TextRepetition, + } +} + +fn low() -> BloatEstimate { + BloatEstimate { + score: 0, + reason: BloatReason::LowSignal, + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn json_rows_estimate_is_redacted() { + let content = r#"[{"secret":"a"},{"secret":"b"},{"secret":"c"}]"#; + let estimate = estimate_bloat(content, ContentKind::Json); + assert_eq!(estimate.reason, BloatReason::JsonRows); + assert!(estimate.score > 0); + assert!(!estimate.reason.as_str().contains("secret")); + } + + #[test] + fn diff_context_estimate_detects_context_heavy_payload() { + let mut diff = String::from("diff --git a/a b/a\n@@ -1,8 +1,8 @@\n"); + for i in 0..80 { + diff.push_str(&format!(" context line {i}\n")); + } + diff.push_str("-old\n+new\n"); + let estimate = estimate_bloat(&diff, ContentKind::Diff); + assert_eq!(estimate.reason, BloatReason::DiffContext); + assert!(estimate.score > 80); + } + + #[test] + fn plain_unique_text_is_low_signal() { + let estimate = estimate_bloat("alpha\nbeta\ngamma", ContentKind::PlainText); + assert_eq!(estimate.reason, BloatReason::LowSignal); + assert_eq!(estimate.score, 0); + } +} diff --git a/src/pipeline/mod.rs b/src/pipeline/mod.rs new file mode 100644 index 0000000..e676349 --- /dev/null +++ b/src/pipeline/mod.rs @@ -0,0 +1,16 @@ +//! Typed compression pipeline primitives. +//! +//! These types make TinyJuice's safety contract explicit for new code: +//! lossless reformats produce ordinary transform output, while lossy offloads +//! must carry a CCR token from a store put result before model-facing text can +//! be emitted. + +mod estimate; +mod report; +mod transform; + +pub use estimate::{BloatEstimate, BloatReason, estimate_bloat}; +pub use report::{PipelineReport, PipelineSkipReason, PipelineStep}; +pub use transform::{ + OffloadOutput, OffloadTransform, PipelineInput, ReformatTransform, TransformOutput, +}; diff --git a/src/pipeline/report.rs b/src/pipeline/report.rs new file mode 100644 index 0000000..7fda641 --- /dev/null +++ b/src/pipeline/report.rs @@ -0,0 +1,102 @@ +use crate::pipeline::BloatEstimate; +use crate::types::{CompressorKind, ContentKind}; + +/// A transform that ran or was considered by the pipeline. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct PipelineStep { + pub name: &'static str, + pub compressor: Option, +} + +/// Redacted reason the pipeline declined to transform content. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum PipelineSkipReason { + RouterDisabled, + BelowMinBytes, + NoCompressor, + NoSavings, + CcrDisabled, + CcrNotRetained, + FooterWouldGrow, + RecoveryTool, + Other(&'static str), +} + +impl PipelineSkipReason { + pub fn as_str(&self) -> &'static str { + match self { + Self::RouterDisabled => "router_disabled", + Self::BelowMinBytes => "below_min_bytes", + Self::NoCompressor => "no_compressor", + Self::NoSavings => "no_savings", + Self::CcrDisabled => "ccr_disabled", + Self::CcrNotRetained => "ccr_not_retained", + Self::FooterWouldGrow => "footer_would_grow", + Self::RecoveryTool => "recovery_tool", + Self::Other(reason) => reason, + } + } +} + +/// Non-sensitive report for pipeline decisions. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct PipelineReport { + pub content_kind: ContentKind, + pub original_bytes: usize, + pub compacted_bytes: usize, + pub bloat_estimate: Option, + pub applied_steps: Vec, + pub skipped_steps: Vec, + pub ccr_tokens: Vec, + pub lossy: bool, + pub skip_reason: Option, +} + +impl PipelineReport { + pub fn applied( + content_kind: ContentKind, + original_bytes: usize, + compacted_bytes: usize, + compressor: CompressorKind, + lossy: bool, + ccr_token: Option, + ) -> Self { + Self { + content_kind, + original_bytes, + compacted_bytes, + bloat_estimate: None, + applied_steps: vec![PipelineStep { + name: "compat_router", + compressor: Some(compressor), + }], + skipped_steps: Vec::new(), + ccr_tokens: ccr_token.into_iter().collect(), + lossy, + skip_reason: None, + } + } + + pub fn passthrough( + content_kind: ContentKind, + original_bytes: usize, + skip_reason: PipelineSkipReason, + ) -> Self { + Self { + content_kind, + original_bytes, + compacted_bytes: original_bytes, + bloat_estimate: None, + applied_steps: Vec::new(), + skipped_steps: Vec::new(), + ccr_tokens: Vec::new(), + lossy: false, + skip_reason: Some(skip_reason), + } + } + + pub fn with_bloat_estimate(mut self, bloat_estimate: BloatEstimate) -> Self { + self.bloat_estimate = Some(bloat_estimate); + self + } +} diff --git a/src/pipeline/transform.rs b/src/pipeline/transform.rs new file mode 100644 index 0000000..1c2b5b7 --- /dev/null +++ b/src/pipeline/transform.rs @@ -0,0 +1,121 @@ +use crate::cache::{CcrPutResult, CcrStore}; +use crate::types::{CompressInput, CompressorKind, ContentKind}; +use std::collections::HashMap; + +/// Input passed to typed transforms. +#[derive(Debug, Clone, Copy)] +pub struct PipelineInput<'a> { + pub content: &'a str, + pub content_kind: ContentKind, + pub original_bytes: usize, +} + +impl<'a> From<&CompressInput<'a>> for PipelineInput<'a> { + fn from(input: &CompressInput<'a>) -> Self { + Self { + content: input.content, + content_kind: input.kind, + original_bytes: input.original_bytes, + } + } +} + +/// Lossless transform output. +#[derive(Debug, Clone)] +pub struct TransformOutput { + pub text: String, + pub kind: CompressorKind, + pub facts: Option>, +} + +impl TransformOutput { + pub fn new(text: String, kind: CompressorKind) -> Self { + Self { + text, + kind, + facts: None, + } + } +} + +/// Lossy transform output with a verified retained CCR token. +#[derive(Debug, Clone)] +pub struct OffloadOutput { + text: String, + kind: CompressorKind, + token: String, + facts: Option>, +} + +impl OffloadOutput { + /// Construct an offload output only from a retained CCR put result. + pub fn from_retained_put( + text: String, + kind: CompressorKind, + put: CcrPutResult, + ) -> Option { + put.retained().then(|| Self { + text, + kind, + token: put.token().to_string(), + facts: None, + }) + } + + pub fn text(&self) -> &str { + &self.text + } + + pub fn kind(&self) -> CompressorKind { + self.kind + } + + pub fn token(&self) -> &str { + &self.token + } + + pub fn facts(&self) -> Option<&HashMap> { + self.facts.as_ref() + } +} + +pub trait ReformatTransform { + fn name(&self) -> &'static str; + fn applies_to(&self, input: &PipelineInput<'_>) -> bool; + fn apply(&self, input: &PipelineInput<'_>) -> Option; +} + +pub trait OffloadTransform { + fn name(&self) -> &'static str; + fn estimate_bloat(&self, input: &PipelineInput<'_>) -> f32; + fn apply(&self, input: &PipelineInput<'_>, store: &dyn CcrStore) -> Option; +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn offload_output_requires_retained_ccr_put() { + let token = "a1b2c3d4".repeat(4); + let rejected = CcrPutResult::new(token.clone(), false); + assert!( + OffloadOutput::from_retained_put( + "partial".to_string(), + CompressorKind::Generic, + rejected, + ) + .is_none() + ); + + let retained = CcrPutResult::new(token.clone(), true); + let out = OffloadOutput::from_retained_put( + "partial".to_string(), + CompressorKind::Generic, + retained, + ) + .expect("retained CCR put constructs offload output"); + assert_eq!(out.text(), "partial"); + assert_eq!(out.token(), token); + } +} diff --git a/src/policy/mod.rs b/src/policy/mod.rs new file mode 100644 index 0000000..6804d75 --- /dev/null +++ b/src/policy/mod.rs @@ -0,0 +1,8 @@ +//! Host-facing compaction policy helpers. + +pub mod shell; + +pub use shell::{ + ShellCompactionPolicy, ShellPolicyDecision, apply_shell_compaction_policy, + is_file_content_inspection_command, +}; diff --git a/src/policy/shell.rs b/src/policy/shell.rs new file mode 100644 index 0000000..106a101 --- /dev/null +++ b/src/policy/shell.rs @@ -0,0 +1,444 @@ +use std::path::Path; + +use crate::types::ToolExecutionInput; + +/// Host policy for shell-command output. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ShellCompactionPolicy { + CompactAll, + SkipAll, + SkipFileContent, + AllowSafeInventory, +} + +/// Redacted decision for shell-command output. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ShellPolicyDecision { + Compact, + SkipAll, + SkipFileContent, + SkipMixedShell, + SkipUnsafeAction, +} + +impl ShellPolicyDecision { + pub fn as_str(self) -> &'static str { + match self { + Self::Compact => "compact", + Self::SkipAll => "skip_all", + Self::SkipFileContent => "skip_file_content", + Self::SkipMixedShell => "skip_mixed_shell", + Self::SkipUnsafeAction => "skip_unsafe_action", + } + } + + pub fn should_compact(self) -> bool { + self == Self::Compact + } +} + +/// Apply shell-output compaction policy to a command-bearing tool input. +pub fn apply_shell_compaction_policy( + input: &ToolExecutionInput, + policy: ShellCompactionPolicy, +) -> ShellPolicyDecision { + match policy { + ShellCompactionPolicy::CompactAll => ShellPolicyDecision::Compact, + ShellCompactionPolicy::SkipAll => ShellPolicyDecision::SkipAll, + ShellCompactionPolicy::SkipFileContent => { + if command_parts(input).is_some_and(|parts| parts.contains_file_content_read()) { + ShellPolicyDecision::SkipFileContent + } else { + ShellPolicyDecision::Compact + } + } + ShellCompactionPolicy::AllowSafeInventory => { + let Some(parts) = command_parts(input) else { + return ShellPolicyDecision::Compact; + }; + if parts.has_mixed_sequence { + return ShellPolicyDecision::SkipMixedShell; + } + if parts.has_unsafe_action() { + return ShellPolicyDecision::SkipUnsafeAction; + } + if parts.contains_file_content_read() { + return ShellPolicyDecision::SkipFileContent; + } + ShellPolicyDecision::Compact + } + } +} + +/// True when the command is a well-known file-content inspection tool. +pub fn is_file_content_inspection_command(input: &ToolExecutionInput) -> bool { + command_parts(input).is_some_and(|parts| parts.contains_file_content_read()) +} + +fn command_parts(input: &ToolExecutionInput) -> Option { + if let Some(command) = input.command.as_deref().filter(|c| !c.trim().is_empty()) { + return parse_command(command); + } + + let argv = input.argv.as_deref()?.to_vec(); + if argv.is_empty() { + return None; + } + Some(CommandParts { + has_redirect: argv.iter().any(|token| is_redirect_token(token)), + segments: vec![argv], + has_mixed_sequence: false, + }) +} + +struct CommandParts { + segments: Vec>, + has_mixed_sequence: bool, + has_redirect: bool, +} + +impl CommandParts { + fn contains_file_content_read(&self) -> bool { + self.segments + .first() + .is_some_and(|tokens| is_file_content_read(tokens)) + } + + fn has_unsafe_action(&self) -> bool { + self.has_redirect + || self + .segments + .iter() + .any(|tokens| contains_unsafe_action(tokens)) + } +} + +fn parse_command(command: &str) -> Option { + let stripped = strip_leading_cd_prefix(command.trim()); + if stripped.is_empty() { + return None; + } + + let has_mixed_sequence = contains_sequence_operator(stripped); + let has_redirect = contains_unquoted_redirect(stripped); + let segments: Vec> = split_unquoted_pipes(stripped) + .into_iter() + .map(crate::reduce::tokenize_command) + .filter(|tokens| !tokens.is_empty()) + .collect(); + + if segments.is_empty() { + None + } else { + Some(CommandParts { + segments, + has_mixed_sequence, + has_redirect, + }) + } +} + +fn strip_leading_cd_prefix(mut command: &str) -> &str { + loop { + let Some((left, right)) = split_once_unquoted_and(command) else { + return command.trim(); + }; + let tokens = crate::reduce::tokenize_command(left); + if tokens.first().map(|s| s.as_str()) != Some("cd") || tokens.len() < 2 { + return command.trim(); + } + command = right.trim(); + } +} + +fn split_once_unquoted_and(s: &str) -> Option<(&str, &str)> { + let bytes = s.as_bytes(); + let mut quote: Option = None; + let mut escaping = false; + let mut i = 0; + while i + 1 < bytes.len() { + let ch = s[i..].chars().next()?; + if escaping { + escaping = false; + i += ch.len_utf8(); + continue; + } + if ch == '\\' { + escaping = true; + i += ch.len_utf8(); + continue; + } + if let Some(q) = quote { + if ch == q { + quote = None; + } + i += ch.len_utf8(); + continue; + } + if ch == '\'' || ch == '"' { + quote = Some(ch); + i += ch.len_utf8(); + continue; + } + if bytes[i] == b'&' && bytes[i + 1] == b'&' { + return Some((&s[..i], &s[i + 2..])); + } + i += ch.len_utf8(); + } + None +} + +fn contains_sequence_operator(s: &str) -> bool { + let bytes = s.as_bytes(); + let mut quote: Option = None; + let mut escaping = false; + let mut i = 0; + while i < bytes.len() { + let ch = s[i..].chars().next().expect("valid char boundary"); + if escaping { + escaping = false; + i += ch.len_utf8(); + continue; + } + if ch == '\\' { + escaping = true; + i += ch.len_utf8(); + continue; + } + if let Some(q) = quote { + if ch == q { + quote = None; + } + i += ch.len_utf8(); + continue; + } + if ch == '\'' || ch == '"' { + quote = Some(ch); + i += ch.len_utf8(); + continue; + } + if ch == ';' || ch == '\n' { + return true; + } + if i + 1 < bytes.len() + && ((bytes[i] == b'&' && bytes[i + 1] == b'&') + || (bytes[i] == b'|' && bytes[i + 1] == b'|')) + { + return true; + } + i += ch.len_utf8(); + } + false +} + +fn contains_unquoted_redirect(s: &str) -> bool { + let mut quote: Option = None; + let mut escaping = false; + for ch in s.chars() { + if escaping { + escaping = false; + continue; + } + if ch == '\\' { + escaping = true; + continue; + } + if let Some(q) = quote { + if ch == q { + quote = None; + } + continue; + } + if ch == '\'' || ch == '"' { + quote = Some(ch); + continue; + } + if ch == '<' || ch == '>' { + return true; + } + } + false +} + +fn split_unquoted_pipes(s: &str) -> Vec<&str> { + let bytes = s.as_bytes(); + let mut out = Vec::new(); + let mut quote: Option = None; + let mut escaping = false; + let mut start = 0; + let mut i = 0; + while i < bytes.len() { + let ch = s[i..].chars().next().expect("valid char boundary"); + if escaping { + escaping = false; + i += ch.len_utf8(); + continue; + } + if ch == '\\' { + escaping = true; + i += ch.len_utf8(); + continue; + } + if let Some(q) = quote { + if ch == q { + quote = None; + } + i += ch.len_utf8(); + continue; + } + if ch == '\'' || ch == '"' { + quote = Some(ch); + i += ch.len_utf8(); + continue; + } + if ch == '|' { + if i + 1 < bytes.len() && bytes[i + 1] == b'|' { + i += 2; + continue; + } + out.push(s[start..i].trim()); + start = i + 1; + } + i += ch.len_utf8(); + } + out.push(s[start..].trim()); + out +} + +fn argv0(tokens: &[String]) -> Option { + tokens.first().map(|cmd| { + Path::new(cmd) + .file_name() + .map(|name| name.to_string_lossy().to_string()) + .unwrap_or_else(|| cmd.to_string()) + }) +} + +fn is_file_content_read(tokens: &[String]) -> bool { + let Some(cmd) = argv0(tokens) else { + return false; + }; + matches!( + cmd.as_str(), + "cat" | "nl" | "bat" | "batcat" | "jq" | "yq" | "head" | "tail" | "sed" + ) +} + +fn contains_unsafe_action(tokens: &[String]) -> bool { + let Some(cmd) = argv0(tokens) else { + return false; + }; + if tokens.iter().any(|token| is_redirect_token(token)) { + return true; + } + match cmd.as_str() { + "find" => tokens.iter().skip(1).any(|t| { + matches!( + t.as_str(), + "-exec" | "-execdir" | "-ok" | "-okdir" | "-delete" + ) + }), + "fd" => tokens.iter().skip(1).any(|t| { + matches!( + t.as_str(), + "-x" | "-X" | "--exec" | "--exec-batch" | "--batch-size" + ) + }), + "rm" | "mv" | "cp" | "install" | "chmod" | "chown" | "ln" | "mkdir" | "rmdir" => true, + "git" => tokens.get(1).is_some_and(|sub| { + matches!( + sub.as_str(), + "checkout" + | "switch" + | "reset" + | "clean" + | "restore" + | "apply" + | "am" + | "merge" + | "rebase" + | "commit" + ) + }), + _ => false, + } +} + +fn is_redirect_token(token: &str) -> bool { + token.contains('>') || token.contains('<') +} + +#[cfg(test)] +mod tests { + use super::*; + + fn input(command: &str) -> ToolExecutionInput { + ToolExecutionInput { + tool_name: "shell".to_owned(), + command: Some(command.to_owned()), + ..Default::default() + } + } + + fn decide(command: &str) -> ShellPolicyDecision { + apply_shell_compaction_policy(&input(command), ShellCompactionPolicy::AllowSafeInventory) + } + + #[test] + fn exact_file_reads_are_skipped() { + assert_eq!( + decide("cat src/lib.rs"), + ShellPolicyDecision::SkipFileContent + ); + assert_eq!( + decide("sed -n '1,200p' src/lib.rs"), + ShellPolicyDecision::SkipFileContent + ); + assert_eq!( + decide("jq . package.json"), + ShellPolicyDecision::SkipFileContent + ); + } + + #[test] + fn inventory_pipelines_are_allowed() { + assert_eq!( + decide("find . -type f | sort | head -n 20"), + ShellPolicyDecision::Compact + ); + assert_eq!(decide("rg --files"), ShellPolicyDecision::Compact); + assert_eq!(decide("git ls-files"), ShellPolicyDecision::Compact); + assert_eq!( + decide("cd crate && rg --files"), + ShellPolicyDecision::Compact + ); + } + + #[test] + fn unsafe_inventory_actions_are_skipped() { + assert_eq!( + decide(r"find . -exec cat {} \;"), + ShellPolicyDecision::SkipUnsafeAction + ); + assert_eq!( + decide("fd --exec cat {}"), + ShellPolicyDecision::SkipUnsafeAction + ); + assert_eq!( + decide("find . -type f > files.txt"), + ShellPolicyDecision::SkipUnsafeAction + ); + } + + #[test] + fn mixed_shell_sequences_are_skipped() { + assert_eq!( + decide("git status; cat src/lib.rs"), + ShellPolicyDecision::SkipMixedShell + ); + assert_eq!( + decide("cat src/lib.rs && git status"), + ShellPolicyDecision::SkipMixedShell + ); + } +} diff --git a/src/reduce.rs b/src/reduce.rs index b02659c..c8bf027 100644 --- a/src/reduce.rs +++ b/src/reduce.rs @@ -10,6 +10,7 @@ use regex::Regex; use crate::{ classify::classify_execution, + policy::{ShellCompactionPolicy, ShellPolicyDecision, apply_shell_compaction_policy}, text::{ clamp_text, clamp_text_middle, count_text_chars, dedupe_adjacent, head_tail, normalize_lines, pluralize, strip_ansi, trim_empty_edges, @@ -98,21 +99,7 @@ pub fn normalize_execution_input(input: ToolExecutionInput) -> ToolExecutionInpu } } -/// True when the command is a well-known file-content inspection tool. -pub fn is_file_content_inspection_command(input: &ToolExecutionInput) -> bool { - static FILE_TOOLS: &[&str] = &[ - "cat", "sed", "head", "tail", "nl", "bat", "batcat", "jq", "yq", - ]; - let argv = input.argv.as_deref().unwrap_or(&[]); - if argv.is_empty() { - return false; - } - let argv0 = std::path::Path::new(&argv[0]) - .file_name() - .map(|n| n.to_string_lossy().to_string()) - .unwrap_or_default(); - FILE_TOOLS.contains(&argv0.as_str()) -} +pub use crate::policy::is_file_content_inspection_command; // --------------------------------------------------------------------------- // Git-status post-processor @@ -785,10 +772,12 @@ pub fn reduce_execution_with_rules( }; } - // File-content inspection commands are never compacted - if classification.matched_reducer.as_deref() == Some("generic/fallback") - && is_file_content_inspection_command(&normalized_input) - { + // Shell output may be summarized only when the host policy says the command + // is safe for compaction. This runs before reducer selection so mixed + // command strings cannot accidentally match a specialised reducer prefix. + let shell_policy_decision = + apply_shell_compaction_policy(&normalized_input, ShellCompactionPolicy::AllowSafeInventory); + if !matches!(shell_policy_decision, ShellPolicyDecision::Compact) { return CompactResult { inline_text: raw_text, preview_text: None, diff --git a/src/reduce_tests.rs b/src/reduce_tests.rs index afbe16e..15815ba 100644 --- a/src/reduce_tests.rs +++ b/src/reduce_tests.rs @@ -278,6 +278,58 @@ fn file_inspection_command_with_path_prefix() { assert!(is_file_content_inspection_command(&input)); } +#[test] +fn safe_inventory_pipeline_may_use_generic_fallback() { + let stdout = (0..80) + .map(|i| format!("src/generated/module_{i}.rs")) + .collect::>() + .join("\n"); + let input = ToolExecutionInput { + tool_name: "bash".to_owned(), + command: Some("find . -type f | sort | head -n 20".to_owned()), + stdout: Some(stdout.clone()), + ..Default::default() + }; + let result = run(input); + assert!( + result.inline_text.len() < stdout.len(), + "got: {}", + result.inline_text + ); +} + +#[test] +fn mixed_shell_sequence_stays_raw() { + let stdout = (0..80) + .map(|i| format!("line {i}")) + .collect::>() + .join("\n"); + let input = ToolExecutionInput { + tool_name: "bash".to_owned(), + command: Some("git status; cat src/lib.rs".to_owned()), + stdout: Some(stdout.clone()), + ..Default::default() + }; + let result = run(input); + assert_eq!(result.inline_text, stdout); +} + +#[test] +fn unsafe_find_exec_stays_raw() { + let stdout = (0..80) + .map(|i| format!("file {i} contents")) + .collect::>() + .join("\n"); + let input = ToolExecutionInput { + tool_name: "bash".to_owned(), + command: Some(r"find . -exec cat {} \;".to_owned()), + stdout: Some(stdout.clone()), + ..Default::default() + }; + let result = run(input); + assert_eq!(result.inline_text, stdout); +} + // --- build_raw_text via reduction pipeline --- #[test] diff --git a/src/types.rs b/src/types.rs index 3ee0871..a5eacb4 100644 --- a/src/types.rs +++ b/src/types.rs @@ -551,13 +551,22 @@ impl Default for CompressOptions { } } -/// The result of the universal [`crate::compress_content`] -/// entry point: the compacted text (with any CCR footer already appended), plus -/// metadata for callers/stats. +/// The result of the universal [`crate::compress_content`] entry point. +/// +/// `text` preserves the original compatibility contract: it is the final +/// model-facing string with any CCR recovery footer already appended. Hosts that +/// apply their own downstream caps should use `body` and `recovery_footer` +/// instead, truncate only `body`, then reattach `recovery_footer` so the +/// recovery marker cannot be severed. #[derive(Debug, Clone)] pub struct CompressedOutput { - /// Final text to inline into context (includes the retrieval footer when lossy). + /// Final text to inline into context (includes the retrieval footer when present). pub text: String, + /// Compacted body without the recovery footer. + pub body: String, + /// Recovery footer to append after host-side truncation, if CCR retained the + /// original. + pub recovery_footer: Option, /// The detected content kind. pub content_kind: ContentKind, /// Which compressor fired (`None` ⇒ pass-through). @@ -579,7 +588,9 @@ impl CompressedOutput { pub fn passthrough(content: String, kind: ContentKind) -> Self { let len = content.len(); Self { - text: content, + text: content.clone(), + body: content, + recovery_footer: None, content_kind: kind, compressor: CompressorKind::None, lossy: false, From e19ac4d995752279f91963a344fa2c1f00b8e80c Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Sun, 5 Jul 2026 05:56:32 +0000 Subject: [PATCH 002/120] Add class-labeled savings records --- README.md | 6 ++ src/compress.rs | 15 +++- src/savings.rs | 226 +++++++++++++++++++++++++++++++++++++++++++++++- 3 files changed, 244 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 8e16f75..64640c9 100644 --- a/README.md +++ b/README.md @@ -84,6 +84,12 @@ tool-call/result boundary alignment, latest user/assistant anchors, JSON string-leaf shrinking, and old tool-result digesting with sensitive metadata redaction. +`savings::configure_record_recorder` installs a metadata-only savings recorder +that receives class-labeled `SavingsRecord` values (`counted`, `measured`, or +`estimated`) with content kind, compressor, byte/token counts, lossy/CCR flags, +and redacted rule or skip labels. The older four-argument +`configure_recorder` callback remains as a compatibility wrapper. + Run the local analytics interface: ```sh diff --git a/src/compress.rs b/src/compress.rs index 82fee21..59c51ba 100644 --- a/src/compress.rs +++ b/src/compress.rs @@ -263,8 +263,19 @@ pub async fn route_with_store_report( ); // Record savings for the dashboard (tokens + cost saved for the LLM the - // result is being compressed for). - savings::record(kind, out.kind, original_tokens, compacted_tokens); + // result is being compressed for). Token counts are estimated; byte counts + // are measured directly from this reducer invocation. + savings::record_event( + savings::SavingsRecord::estimated_compaction( + kind, + out.kind, + original_tokens, + compacted_tokens, + ) + .with_bytes(original_bytes as u64, compacted_bytes as u64) + .with_lossy(out.lossy) + .with_ccr_token_present(ccr_token.is_some()), + ); let res = CompressedOutput { text, diff --git a/src/savings.rs b/src/savings.rs index 2c94122..d123a01 100644 --- a/src/savings.rs +++ b/src/savings.rs @@ -8,18 +8,138 @@ use std::sync::{Arc, OnceLock, RwLock}; use crate::types::{CompressorKind, ContentKind}; +/// Accounting confidence for a savings event. +/// +/// The class describes the numbers in the record, not the compressor itself: +/// reducer byte counts are measured, token counts from [`crate::tokens`] are +/// estimates, and call-collapse events are counted. +#[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum AccountingClass { + Counted, + Measured, + Estimated, +} + +impl AccountingClass { + pub fn as_str(self) -> &'static str { + match self { + Self::Counted => "counted", + Self::Measured => "measured", + Self::Estimated => "estimated", + } + } +} + +/// Provider-reported usage fields hosts may attach when real model usage is +/// available. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, serde::Serialize, serde::Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ModelUsage { + pub input_tokens: u64, + pub output_tokens: u64, + pub cache_read_tokens: u64, + pub cache_creation_tokens: u64, +} + +/// Non-sensitive attribution record for a TokenJuice savings event. +#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct SavingsRecord { + pub accounting_class: AccountingClass, + pub content_kind: ContentKind, + pub compressor: CompressorKind, + pub original_tokens: u64, + pub compacted_tokens: u64, + pub original_bytes: Option, + pub compacted_bytes: Option, + pub lossy: bool, + pub ccr_token_present: bool, + pub rule_id: Option, + pub skip_reason: Option, + pub measured_usage: Option, +} + +impl SavingsRecord { + pub fn estimated_compaction( + content_kind: ContentKind, + compressor: CompressorKind, + original_tokens: u64, + compacted_tokens: u64, + ) -> Self { + Self { + accounting_class: AccountingClass::Estimated, + content_kind, + compressor, + original_tokens, + compacted_tokens, + original_bytes: None, + compacted_bytes: None, + lossy: false, + ccr_token_present: false, + rule_id: None, + skip_reason: None, + measured_usage: None, + } + } + + pub fn with_bytes(mut self, original_bytes: u64, compacted_bytes: u64) -> Self { + self.original_bytes = Some(original_bytes); + self.compacted_bytes = Some(compacted_bytes); + self + } + + pub fn with_lossy(mut self, lossy: bool) -> Self { + self.lossy = lossy; + self + } + + pub fn with_ccr_token_present(mut self, ccr_token_present: bool) -> Self { + self.ccr_token_present = ccr_token_present; + self + } + + pub fn with_rule_id(mut self, rule_id: impl Into) -> Self { + self.rule_id = Some(rule_id.into()); + self + } + + pub fn with_skip_reason(mut self, skip_reason: impl Into) -> Self { + self.skip_reason = Some(skip_reason.into()); + self + } + + pub fn with_measured_usage(mut self, measured_usage: ModelUsage) -> Self { + self.measured_usage = Some(measured_usage); + self + } +} + pub type SavingsRecorder = dyn Fn(ContentKind, CompressorKind, u64, u64) + Send + Sync + 'static; +pub type SavingsRecordRecorder = dyn Fn(SavingsRecord) + Send + Sync + 'static; fn recorder_cell() -> &'static RwLock>> { static RECORDER: OnceLock>>> = OnceLock::new(); RECORDER.get_or_init(|| RwLock::new(None)) } +fn record_recorder_cell() -> &'static RwLock>> { + static RECORDER: OnceLock>>> = OnceLock::new(); + RECORDER.get_or_init(|| RwLock::new(None)) +} + /// Install or clear the host savings recorder. pub fn configure_recorder(recorder: Option>) { *recorder_cell().write().unwrap_or_else(|p| p.into_inner()) = recorder; } +/// Install or clear the host recorder for rich, class-labeled savings records. +pub fn configure_record_recorder(recorder: Option>) { + *record_recorder_cell() + .write() + .unwrap_or_else(|p| p.into_inner()) = recorder; +} + /// Record one compaction event. No-op when no host recorder is installed. pub fn record( content_kind: ContentKind, @@ -27,11 +147,115 @@ pub fn record( original_tokens: u64, compacted_tokens: u64, ) { + record_event(SavingsRecord::estimated_compaction( + content_kind, + compressor, + original_tokens, + compacted_tokens, + )); +} + +/// Record one rich savings event. No-op when no host recorder is installed. +pub fn record_event(record: SavingsRecord) { + let record_recorder = record_recorder_cell() + .read() + .unwrap_or_else(|p| p.into_inner()) + .clone(); + if let Some(recorder) = record_recorder { + recorder(record.clone()); + } + let recorder = recorder_cell() .read() .unwrap_or_else(|p| p.into_inner()) .clone(); if let Some(recorder) = recorder { - recorder(content_kind, compressor, original_tokens, compacted_tokens); + recorder( + record.content_kind, + record.compressor, + record.original_tokens, + record.compacted_tokens, + ); + } +} + +#[cfg(test)] +mod tests { + use std::sync::Mutex; + + use super::*; + + #[test] + fn rich_record_serializes_only_metadata() { + let record = + SavingsRecord::estimated_compaction(ContentKind::Log, CompressorKind::Log, 120, 40) + .with_bytes(480, 160) + .with_lossy(true) + .with_ccr_token_present(true) + .with_rule_id("cargo-test") + .with_skip_reason("no_raw_content_here") + .with_measured_usage(ModelUsage { + input_tokens: 10, + output_tokens: 20, + cache_read_tokens: 5, + cache_creation_tokens: 7, + }); + + let json = serde_json::to_string(&record).expect("record serializes"); + assert!(json.contains("\"accountingClass\":\"estimated\"")); + assert!(json.contains("\"contentKind\":\"log\"")); + assert!(json.contains("\"compressor\":\"log\"")); + assert!(!json.contains("secret tool output")); + assert!(!json.contains("prompt")); + } + + #[test] + fn record_event_invokes_rich_and_legacy_recorders() { + let rich_records = Arc::new(Mutex::new(Vec::new())); + let legacy_records = Arc::new(Mutex::new(Vec::new())); + + let rich_records_clone = Arc::clone(&rich_records); + configure_record_recorder(Some(Arc::new(move |record| { + rich_records_clone + .lock() + .expect("rich records lock") + .push(record); + }))); + + let legacy_records_clone = Arc::clone(&legacy_records); + configure_recorder(Some(Arc::new( + move |kind, compressor, original, compacted| { + legacy_records_clone + .lock() + .expect("legacy records lock") + .push((kind, compressor, original, compacted)); + }, + ))); + + record_event( + SavingsRecord::estimated_compaction( + ContentKind::Json, + CompressorKind::SmartCrusher, + 100, + 25, + ) + .with_bytes(400, 100) + .with_ccr_token_present(true), + ); + + configure_record_recorder(None); + configure_recorder(None); + + let rich = rich_records.lock().expect("rich records lock"); + assert_eq!(rich.len(), 1); + assert_eq!(rich[0].accounting_class, AccountingClass::Estimated); + assert_eq!(rich[0].original_bytes, Some(400)); + assert!(rich[0].ccr_token_present); + + let legacy = legacy_records.lock().expect("legacy records lock"); + assert_eq!( + legacy.as_slice(), + &[(ContentKind::Json, CompressorKind::SmartCrusher, 100, 25)] + ); } } From f2fc10f374bbde7749d144b04bf864d36bd13ebb Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Sun, 5 Jul 2026 06:54:23 +0000 Subject: [PATCH 003/120] Add web extract truncate-store reducer --- README.md | 6 + src/compressors/mod.rs | 1 + src/compressors/web_extract.rs | 450 +++++++++++++++++++++++++++++++++ src/lib.rs | 6 + src/types.rs | 104 ++++++++ 5 files changed, 567 insertions(+) create mode 100644 src/compressors/web_extract.rs diff --git a/README.md b/README.md index 64640c9..d422348 100644 --- a/README.md +++ b/README.md @@ -90,6 +90,12 @@ that receives class-labeled `SavingsRecord` values (`counted`, `measured`, or and redacted rule or skip labels. The older four-argument `configure_recorder` callback remains as a compatibility wrapper. +Already-extracted web pages can be passed through `reduce_web_extract` or +`reduce_web_extract_with_store`. The reducer removes inline base64 image blobs, +preserves ordinary markdown image URLs, and stores omitted middle content in CCR +before emitting a head/tail truncation footer. If CCR cannot retain the full +cleaned page, the reducer returns the cleaned page without lossy truncation. + Run the local analytics interface: ```sh diff --git a/src/compressors/mod.rs b/src/compressors/mod.rs index ddb5f8d..f5b0b74 100644 --- a/src/compressors/mod.rs +++ b/src/compressors/mod.rs @@ -17,6 +17,7 @@ pub mod log; pub mod ml_text; pub mod search; pub mod signals; +pub mod web_extract; use async_trait::async_trait; diff --git a/src/compressors/web_extract.rs b/src/compressors/web_extract.rs new file mode 100644 index 0000000..9c86102 --- /dev/null +++ b/src/compressors/web_extract.rs @@ -0,0 +1,450 @@ +//! Reducer for already-extracted web pages. +//! +//! TinyJuice does not fetch URLs. Hosts pass provider-cleaned markdown/text/HTML +//! here, and the reducer bounds large pages with a recoverable head/tail window +//! backed by CCR. + +use regex::Regex; + +use crate::cache::{self, CcrStore}; +use crate::types::{ + WebExtractBatchInput, WebExtractOptions, WebExtractReduceInput, WebExtractReduction, +}; + +const FOOTER_RULE: &str = "-------- [TOKENJUICE WEB TRUNCATED] --------"; + +/// Reduce one extracted web page with the global CCR store. +pub fn reduce_web_extract( + input: &WebExtractReduceInput, + options: &WebExtractOptions, +) -> WebExtractReduction { + reduce_web_extract_with_store(input, options, &cache::GlobalCcrStore) +} + +/// Reduce one extracted web page with an injected CCR store. +pub fn reduce_web_extract_with_store( + input: &WebExtractReduceInput, + options: &WebExtractOptions, + store: &dyn CcrStore, +) -> WebExtractReduction { + let (clean, replaced) = clean_content(&input.content, options.convert_base64_images); + let original_chars = clean.chars().count(); + let limit = clamp_limit(input.char_limit.unwrap_or(options.char_limit), options); + let source_url_hash = cache::short_hash(&input.url); + let source_host = source_host(&input.url); + + if original_chars <= limit { + let inline_chars = clean.chars().count(); + return WebExtractReduction { + text: clean.clone(), + body: clean, + recovery_footer: None, + ccr_token: None, + source_host, + source_url_hash, + title: input.title.clone(), + format: input.format, + original_chars, + inline_chars, + head_chars: inline_chars, + tail_chars: 0, + omitted_chars: 0, + truncated: false, + full_text_retained: false, + base64_images_replaced: replaced, + }; + } + + let put = store.put(&clean); + if !put.retained() { + // Hard repository invariant: do not emit unrecoverable lossy output. + let inline_chars = clean.chars().count(); + return WebExtractReduction { + text: clean.clone(), + body: clean, + recovery_footer: None, + ccr_token: None, + source_host, + source_url_hash, + title: input.title.clone(), + format: input.format, + original_chars, + inline_chars, + head_chars: inline_chars, + tail_chars: 0, + omitted_chars: 0, + truncated: false, + full_text_retained: false, + base64_images_replaced: replaced, + }; + } + + let head_budget = ((limit as f32) * options.head_ratio.clamp(0.1, 0.9)).floor() as usize; + let tail_budget = limit.saturating_sub(head_budget).max(1); + let head = snap_head_to_line(&char_prefix(&clean, head_budget), head_budget); + let tail = snap_tail_to_line(&char_suffix(&clean, tail_budget), tail_budget); + let head_chars = head.chars().count(); + let tail_chars = tail.chars().count(); + let omitted_chars = original_chars.saturating_sub(head_chars + tail_chars); + let omitted_start_line = head.lines().count().saturating_add(1); + + let mut body = String::with_capacity(head.len() + tail.len() + 160); + body.push_str(&head); + if !body.ends_with('\n') { + body.push('\n'); + } + body.push_str("\n-------- [OMITTED MIDDLE] --------\n"); + body.push_str(&tail); + + let footer = web_recovery_footer( + put.token(), + original_chars, + head_chars, + tail_chars, + omitted_start_line, + ); + let mut text = body.clone(); + text.push_str(&footer); + + WebExtractReduction { + inline_chars: text.chars().count(), + text, + body, + recovery_footer: Some(footer), + ccr_token: Some(put.token().to_string()), + source_host, + source_url_hash, + title: input.title.clone(), + format: input.format, + original_chars, + head_chars, + tail_chars, + omitted_chars, + truncated: true, + full_text_retained: true, + base64_images_replaced: replaced, + } +} + +/// Reduce a batch while preserving per-page retrieval footers. +pub fn reduce_web_extract_batch_with_store( + input: &WebExtractBatchInput, + options: &WebExtractOptions, + store: &dyn CcrStore, +) -> Vec { + let mut page_options = *options; + if let Some(default_char_limit) = input.default_char_limit { + page_options.char_limit = default_char_limit; + } + if let Some(max_combined_inline_chars) = input.max_combined_inline_chars { + page_options.max_combined_inline_chars = max_combined_inline_chars; + } + let mut reductions: Vec = input + .pages + .iter() + .map(|page| reduce_web_extract_with_store(page, &page_options, store)) + .collect(); + + if combined_inline_chars(&reductions) <= page_options.max_combined_inline_chars + || input.pages.is_empty() + { + return reductions; + } + + let per_page_limit = (page_options.max_combined_inline_chars / input.pages.len()) + .max(page_options.min_char_limit) + .min(page_options.char_limit); + let tightened_options = WebExtractOptions { + char_limit: per_page_limit, + ..page_options + }; + reductions = input + .pages + .iter() + .map(|page| reduce_web_extract_with_store(page, &tightened_options, store)) + .collect(); + + // Never enforce the combined budget by slicing text after reduction; that + // could sever a CCR footer and make omitted text unreachable. + reductions +} + +fn combined_inline_chars(reductions: &[WebExtractReduction]) -> usize { + reductions.iter().map(|r| r.inline_chars).sum() +} + +fn clean_content(content: &str, convert_base64_images: bool) -> (String, usize) { + if !convert_base64_images { + return (content.to_string(), 0); + } + replace_inline_base64_images(content) +} + +/// Replace embedded image bytes while preserving ordinary remote image URLs. +pub fn replace_inline_base64_images(content: &str) -> (String, usize) { + static MARKDOWN_IMAGE: std::sync::OnceLock = std::sync::OnceLock::new(); + static PAREN_IMAGE: std::sync::OnceLock = std::sync::OnceLock::new(); + static RAW_IMAGE: std::sync::OnceLock = std::sync::OnceLock::new(); + + let markdown = MARKDOWN_IMAGE.get_or_init(|| { + Regex::new(r"!\[([^\]]*)\]\(data:image/[A-Za-z0-9.+-]+;base64,[^)]+\)").unwrap() + }); + let paren = PAREN_IMAGE + .get_or_init(|| Regex::new(r"\(data:image/[A-Za-z0-9.+-]+;base64,[^)]+\)").unwrap()); + let raw = RAW_IMAGE.get_or_init(|| { + Regex::new(r"data:image/[A-Za-z0-9.+-]+;base64,[A-Za-z0-9+/=_-]+").unwrap() + }); + + let mut count = 0usize; + let text = markdown.replace_all(content, |caps: ®ex::Captures<'_>| { + count += 1; + let alt = caps.get(1).map(|m| m.as_str().trim()).unwrap_or_default(); + if alt.is_empty() { + "[IMAGE]".to_string() + } else { + format!("[IMAGE: {alt}]") + } + }); + let text = paren.replace_all(&text, |_caps: ®ex::Captures<'_>| { + count += 1; + "[IMAGE]".to_string() + }); + let text = raw.replace_all(&text, |_caps: ®ex::Captures<'_>| { + count += 1; + "[IMAGE]".to_string() + }); + (text.into_owned(), count) +} + +fn clamp_limit(limit: usize, options: &WebExtractOptions) -> usize { + let min = options.min_char_limit.min(options.max_char_limit); + let max = options.max_char_limit.max(min); + limit.clamp(min, max) +} + +fn char_prefix(s: &str, char_count: usize) -> String { + s.chars().take(char_count).collect() +} + +fn char_suffix(s: &str, char_count: usize) -> String { + let chars: Vec = s.chars().collect(); + let start = chars.len().saturating_sub(char_count); + chars[start..].iter().collect() +} + +fn snap_head_to_line(head: &str, budget: usize) -> String { + let Some(idx) = head.rfind('\n') else { + return head.to_string(); + }; + let snapped = &head[..idx + 1]; + if snapped.chars().count() >= budget / 2 { + snapped.to_string() + } else { + head.to_string() + } +} + +fn snap_tail_to_line(tail: &str, budget: usize) -> String { + let Some(idx) = tail.find('\n') else { + return tail.to_string(); + }; + let snapped = &tail[idx + 1..]; + if snapped.chars().count() >= budget / 2 { + snapped.to_string() + } else { + tail.to_string() + } +} + +fn web_recovery_footer( + token: &str, + original_chars: usize, + head_chars: usize, + tail_chars: usize, + omitted_start_line: usize, +) -> String { + let marker = cache::format_marker(token); + format!( + "\n\n{FOOTER_RULE}\n\ + Showing {head_chars} chars (head) + {tail_chars} chars (tail) of \ + {original_chars} total clean characters.\n\ + Full text token: {token} (marker {marker}).\n\ + To read the omitted middle: {} token=\"{token}\" offset={omitted_start_line} limit=\n\ + ----------------------------------------------", + cache::RETRIEVE_TOOL_NAME + ) +} + +fn source_host(url: &str) -> Option { + let after_scheme = url + .split_once("://") + .map(|(_, rest)| rest) + .unwrap_or(url) + .trim_start_matches('/'); + let host = after_scheme + .split(['/', '?', '#']) + .next() + .unwrap_or_default() + .trim(); + if host.is_empty() { + None + } else { + Some(host.to_ascii_lowercase()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::cache::{MemoryCcrStore, parse_markers}; + use crate::types::WebExtractFormat; + use serde_json::Map; + + fn page(content: String) -> WebExtractReduceInput { + WebExtractReduceInput { + url: "https://example.com/path?token=secret".to_string(), + title: Some("Example".to_string()), + content, + format: WebExtractFormat::Markdown, + provider: Some("test".to_string()), + char_limit: None, + metadata: Map::new(), + } + } + + fn options() -> WebExtractOptions { + WebExtractOptions { + char_limit: 80, + min_char_limit: 20, + max_char_limit: 500, + head_ratio: 0.75, + convert_base64_images: true, + max_combined_inline_chars: 1000, + } + } + + #[test] + fn small_page_returns_cleaned_content_without_footer() { + let store = MemoryCcrStore::new(10, 10_000); + let input = page( + "intro ![chart](data:image/png;base64,AAAA1111) ![remote](https://x/y.png)".to_string(), + ); + + let out = reduce_web_extract_with_store(&input, &options(), &store); + + assert_eq!(out.text, "intro [IMAGE: chart] ![remote](https://x/y.png)"); + assert_eq!(out.base64_images_replaced, 1); + assert!(out.recovery_footer.is_none()); + assert!(!out.text.contains("AAAA1111")); + assert!(out.text.contains("https://x/y.png")); + } + + #[test] + fn large_page_returns_head_tail_footer_and_retains_full_text() { + let store = MemoryCcrStore::new(10, 100_000); + let input = page((0..40).map(|i| format!("line-{i:02}\n")).collect()); + + let out = reduce_web_extract_with_store(&input, &options(), &store); + + assert!(out.truncated); + assert!(out.full_text_retained); + assert!(out.text.contains(FOOTER_RULE)); + assert!(out.text.contains("tokenjuice_retrieve")); + assert_eq!( + parse_markers(&out.text), + vec![out.ccr_token.clone().unwrap()] + ); + assert_eq!( + store.get(out.ccr_token.as_deref().unwrap()).as_deref(), + Some(input.content.as_str()) + ); + assert!(out.text.contains("line-00")); + assert!(out.text.contains("line-39")); + } + + #[test] + fn store_failure_returns_cleaned_whole_page_without_unrecoverable_footer() { + let store = MemoryCcrStore::new(1, 16); + let input = page("x".repeat(200)); + + let out = reduce_web_extract_with_store(&input, &options(), &store); + + assert!(!out.truncated); + assert!(!out.full_text_retained); + assert!(out.recovery_footer.is_none()); + assert_eq!(out.text, input.content); + } + + #[test] + fn clamps_invalid_limits() { + let store = MemoryCcrStore::new(10, 100_000); + let mut input = page("x\n".repeat(300)); + input.char_limit = Some(1); + + let out = reduce_web_extract_with_store(&input, &options(), &store); + + assert!(out.head_chars + out.tail_chars >= 20 / 2); + assert!(out.truncated); + } + + #[test] + fn defaults_and_huge_limits_are_clamped() { + let store = MemoryCcrStore::new(10, 100_000); + let mut input = page("x\n".repeat(300)); + let options = WebExtractOptions { + char_limit: 30, + min_char_limit: 20, + max_char_limit: 50, + ..options() + }; + + let defaulted = reduce_web_extract_with_store(&input, &options, &store); + assert!(defaulted.truncated); + assert!(defaulted.head_chars + defaulted.tail_chars <= 30); + + input.char_limit = Some(usize::MAX); + let clamped = reduce_web_extract_with_store(&input, &options, &store); + assert!(clamped.truncated); + assert!(clamped.head_chars + clamped.tail_chars <= 50); + } + + #[test] + fn batch_tightens_budget_without_dropping_recovery_footers() { + let store = MemoryCcrStore::new(10, 100_000); + let input = WebExtractBatchInput { + pages: vec![page("a\n".repeat(200)), page("b\n".repeat(200))], + default_char_limit: Some(80), + max_combined_inline_chars: Some(80), + }; + let options = WebExtractOptions { + min_char_limit: 20, + ..options() + }; + + let out = reduce_web_extract_batch_with_store(&input, &options, &store); + + assert_eq!(out.len(), 2); + assert!(out.iter().all(|r| r.truncated)); + assert!(out.iter().all(|r| r.text.contains(FOOTER_RULE))); + assert!(out.iter().all(|r| parse_markers(&r.text).len() == 1)); + assert!( + out.iter() + .map(|r| r.head_chars + r.tail_chars) + .sum::() + <= 80 + ); + } + + #[test] + fn metadata_uses_host_and_url_hash_not_full_url() { + let store = MemoryCcrStore::new(10, 10_000); + let input = page("short".to_string()); + let out = reduce_web_extract_with_store(&input, &options(), &store); + let json = serde_json::to_string(&out).unwrap(); + + assert_eq!(out.source_host.as_deref(), Some("example.com")); + assert!(!json.contains("token=secret")); + assert!(!json.contains("/path")); + assert!(!out.source_url_hash.is_empty()); + } +} diff --git a/src/lib.rs b/src/lib.rs index c80d08f..ae51171 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -33,6 +33,10 @@ pub use compress::{ pub use compressor::{ CompressionInput, CompressionOutput, CompressionReport, Compressor, PassthroughCompressor, }; +pub use compressors::web_extract::{ + reduce_web_extract, reduce_web_extract_batch_with_store, reduce_web_extract_with_store, + replace_inline_base64_images, +}; pub use compressors::{compressor_for, generic_compressor}; pub use config::CompressionConfig; pub use conversation::{ @@ -59,6 +63,8 @@ pub use tool_integration::{ pub use types::{ AgentTokenjuiceCompression, CompactResult, CompressInput, CompressOptions, CompressOutput, CompressedOutput, CompressorKind, ContentHint, ContentKind, ReduceOptions, ToolExecutionInput, + WebExtractBatchInput, WebExtractFormat, WebExtractOptions, WebExtractReduceInput, + WebExtractReduction, }; #[cfg(test)] diff --git a/src/types.rs b/src/types.rs index a5eacb4..3d7b10c 100644 --- a/src/types.rs +++ b/src/types.rs @@ -395,6 +395,110 @@ pub struct ContentHint { pub explicit: Option, } +// --------------------------------------------------------------------------- +// Web extraction reducer inputs +// --------------------------------------------------------------------------- + +/// Format of already-extracted web content handed to the web reducer. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum WebExtractFormat { + #[default] + Markdown, + Text, + Html, +} + +impl WebExtractFormat { + pub fn as_str(self) -> &'static str { + match self { + Self::Markdown => "markdown", + Self::Text => "text", + Self::Html => "html", + } + } +} + +/// One already-extracted web page to reduce. Hosts own fetching and URL +/// validation; TinyJuice receives only extracted content plus metadata. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct WebExtractReduceInput { + pub url: String, + #[serde(default)] + pub title: Option, + pub content: String, + #[serde(default)] + pub format: WebExtractFormat, + #[serde(default)] + pub provider: Option, + #[serde(default)] + pub char_limit: Option, + #[serde(default)] + pub metadata: serde_json::Map, +} + +/// Batch shape for multi-URL extractors. The reducer keeps page footers intact +/// even when the combined output exceeds the inline budget. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct WebExtractBatchInput { + pub pages: Vec, + #[serde(default)] + pub default_char_limit: Option, + #[serde(default)] + pub max_combined_inline_chars: Option, +} + +/// Web extraction truncation knobs. Defaults match the P1-2 plan. +#[derive(Debug, Clone, Copy, PartialEq)] +pub struct WebExtractOptions { + pub char_limit: usize, + pub min_char_limit: usize, + pub max_char_limit: usize, + pub head_ratio: f32, + pub convert_base64_images: bool, + pub max_combined_inline_chars: usize, +} + +impl Default for WebExtractOptions { + fn default() -> Self { + Self { + char_limit: 15_000, + min_char_limit: 2_000, + max_char_limit: 500_000, + head_ratio: 0.75, + convert_base64_images: true, + max_combined_inline_chars: 100_000, + } + } +} + +/// Metadata-only reduction report for a web page. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct WebExtractReduction { + pub text: String, + pub body: String, + #[serde(default)] + pub recovery_footer: Option, + #[serde(default)] + pub ccr_token: Option, + pub source_host: Option, + pub source_url_hash: String, + #[serde(default)] + pub title: Option, + pub format: WebExtractFormat, + pub original_chars: usize, + pub inline_chars: usize, + pub head_chars: usize, + pub tail_chars: usize, + pub omitted_chars: usize, + pub truncated: bool, + pub full_text_retained: bool, + pub base64_images_replaced: usize, +} + impl ContentHint { /// Convenience: a hint carrying only the producing tool name. pub fn for_tool(tool_name: impl Into) -> Self { From 8b2d7c8e8f79989a72adbe797af973c4d0b4f09b Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Sun, 5 Jul 2026 07:38:55 +0000 Subject: [PATCH 004/120] Add explicit code stub read intent --- README.md | 5 + src/compress.rs | 41 ++++ src/compressors/code.rs | 430 ++++++++++++++++++++++++++++++++++++++- src/lib.rs | 6 +- src/types.rs | 83 ++++++++ tests/e2e_tool_output.rs | 25 ++- 6 files changed, 582 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index d422348..ddf35b2 100644 --- a/README.md +++ b/README.md @@ -96,6 +96,11 @@ preserves ordinary markdown image URLs, and stores omitted middle content in CCR before emitting a head/tail truncation footer. If CCR cannot retain the full cleaned page, the reducer returns the cleaned page without lossy truncation. +Source-code file reads are exact by default. Hosts that intentionally want a +structural view can call `stub_code` with a `StubMode`; the returned +`CodeStubOutput` includes symbols, elided line ranges, and whether tree-sitter +or the heuristic fallback produced the stub. + Run the local analytics interface: ```sh diff --git a/src/compress.rs b/src/compress.rs index 59c51ba..9afdfe2 100644 --- a/src/compress.rs +++ b/src/compress.rs @@ -120,6 +120,16 @@ pub async fn route_with_store_report( let kind = detect_content_kind(content, input.hint); let bloat_estimate = estimate_bloat(content, kind); + if is_exact_file_read(input.hint) { + let res = CompressedOutput::passthrough(content.to_string(), kind); + let report = PipelineReport::passthrough( + kind, + original_bytes, + PipelineSkipReason::Other("exact_file_read"), + ) + .with_bloat_estimate(bloat_estimate); + return (res, report); + } let shell_policy_decision = apply_shell_compaction_policy( &crate::types::ToolExecutionInput { tool_name: input @@ -307,6 +317,14 @@ pub fn detect_only(content: &str, hint: &ContentHint) -> ContentKind { detect_content_kind(content, hint) } +fn is_exact_file_read(hint: &ContentHint) -> bool { + matches!(hint.read_intent, crate::types::ReadIntent::Exact) + && hint + .source_tool + .as_deref() + .is_some_and(|tool| matches!(tool, "file_read" | "read_file" | "fs_read")) +} + #[cfg(test)] mod tests { use super::*; @@ -465,6 +483,29 @@ mod tests { ); } + #[tokio::test] + async fn file_read_hint_is_exact_by_default_even_with_code_extension() { + let content = (0..120) + .map(|i| format!("pub fn generated_{i}() {{ println!(\"{i}\"); }}")) + .collect::>() + .join("\n"); + let hint = ContentHint { + source_tool: Some("file_read".to_owned()), + extension: Some("rs".to_owned()), + ..Default::default() + }; + let store = cache::MemoryCcrStore::new(10, 1_000_000); + let (res, report) = + compress_content_with_store_report(&content, Some(hint), &opts(), &store).await; + + assert!(!res.applied); + assert_eq!(res.text, content); + assert_eq!( + report.skip_reason, + Some(PipelineSkipReason::Other("exact_file_read")) + ); + } + #[tokio::test] async fn lossy_output_declines_when_injected_store_cannot_retain() { let store = cache::MemoryCcrStore::new(10, 16); diff --git a/src/compressors/code.rs b/src/compressors/code.rs index 0cdab43..7bb31ba 100644 --- a/src/compressors/code.rs +++ b/src/compressors/code.rs @@ -16,7 +16,10 @@ use async_trait::async_trait; use std::fmt::Write as _; use super::Compressor; -use crate::types::{CompressInput, CompressOptions, CompressOutput, CompressorKind}; +use crate::types::{ + CodeElision, CodeStubOutput, CompressInput, CompressOptions, CompressOutput, CompressorKind, + LineRange, ParseStatus, ReadIntent, StubMode, SymbolSummary, +}; /// Bodies with more than this many collapsed lines get a placeholder; shorter /// ones are kept verbatim (collapsing tiny bodies isn't worth the marker). @@ -38,6 +41,19 @@ impl Compressor for CodeCompressor { input: &CompressInput<'_>, _opts: &CompressOptions, ) -> Option { + if let ReadIntent::Stub(mode) = &input.hint.read_intent { + let stub = stub_code( + input.content, + input.hint.extension.as_deref(), + mode, + input.original_bytes, + ); + if stub.text.len() < input.content.len() { + return Some(CompressOutput::lossy(stub.text, CompressorKind::Code)); + } + return None; + } + // Prefer the AST path when a grammar matches the file's language; fall // back to the language-agnostic heuristic otherwise (or when the AST // path doesn't shrink the content). @@ -113,6 +129,281 @@ pub fn compress_heuristic(content: &str) -> Option { Some(CompressOutput::lossy(out, CompressorKind::Code)) } +/// Build a structural source-code stub with elision metadata. +/// +/// Hosts call this directly for explicit stub reads. Exact reads should not use +/// this path. +pub fn stub_code( + content: &str, + extension: Option<&str>, + mode: &StubMode, + max_bytes: usize, +) -> CodeStubOutput { + #[cfg(feature = "tokenjuice-treesitter")] + if let Some(ext) = extension + && let Some(out) = treesitter::stub(content, ext, mode) + { + return enforce_stub_budget(out, max_bytes); + } + + enforce_stub_budget(stub_heuristic(content, mode), max_bytes) +} + +#[derive(Debug, Clone)] +struct StubBody { + body_start: usize, + body_end: usize, + body_range: LineRange, + symbol: SymbolSummary, +} + +fn should_expand_body(mode: &StubMode, body: &StubBody) -> bool { + match mode { + StubMode::SignaturesOnly | StubMode::PublicApi => false, + StubMode::MatchedSymbols(symbols) => symbols.iter().any(|s| { + let wanted = s.trim(); + !wanted.is_empty() && wanted == body.symbol.name + }), + StubMode::ExpandAroundLines(ranges) => ranges + .iter() + .copied() + .any(|range| range.intersects(body.body_range)), + } +} + +fn placeholder_for_body(content: &str, body: &StubBody) -> String { + let body_text = &content[body.body_start..body.body_end]; + let lines = body.body_range.end.saturating_sub(body.body_range.start) + 1; + if body_text.trim_start().starts_with('{') { + format!( + "{{ /* ... lines {}-{} elided ({lines} line(s)) ... */ }}", + body.body_range.start, body.body_range.end + ) + } else { + format!( + "... # lines {}-{} elided ({lines} line(s))", + body.body_range.start, body.body_range.end + ) + } +} + +fn build_stub_from_bodies( + content: &str, + bodies: Vec, + mode: &StubMode, + parse_status: ParseStatus, +) -> CodeStubOutput { + let mut out = String::with_capacity(content.len() / 2 + 128); + let mut cursor = 0usize; + let mut symbols = Vec::new(); + let mut elisions = Vec::new(); + + for body in bodies { + if body.body_start < cursor { + continue; + } + symbols.push(body.symbol.clone()); + out.push_str(&content[cursor..body.body_start]); + let body_text = &content[body.body_start..body.body_end]; + let body_lines = body_text.lines().count(); + if body_lines < MIN_BODY_LINES_TO_COLLAPSE || should_expand_body(mode, &body) { + out.push_str(body_text); + } else { + out.push_str(&placeholder_for_body(content, &body)); + elisions.push(CodeElision { + start_line: body.body_range.start, + end_line: body.body_range.end, + reason: format!("body_stub:{:?}", mode), + }); + } + cursor = body.body_end; + } + out.push_str(&content[cursor..]); + + CodeStubOutput { + text: out.trim_end().to_string(), + symbols, + elisions, + parse_status, + } +} + +fn stub_heuristic(content: &str, mode: &StubMode) -> CodeStubOutput { + let mut out = String::with_capacity(content.len() / 2 + 128); + let mut depth: i32 = 0; + let mut collapsed: Vec<(usize, &str)> = Vec::new(); + let mut elisions = Vec::new(); + let mut symbols = Vec::new(); + + let flush = + |out: &mut String, collapsed: &mut Vec<(usize, &str)>, elisions: &mut Vec| { + if collapsed.is_empty() { + return; + } + if collapsed.len() >= MIN_BODY_LINES_TO_COLLAPSE { + let start = collapsed.first().map(|(n, _)| *n).unwrap_or(1); + let end = collapsed.last().map(|(n, _)| *n).unwrap_or(start); + let _ = writeln!( + out, + " {{ /* ... lines {start}-{end} elided ({} line(s)) ... */ }}", + collapsed.len() + ); + elisions.push(CodeElision { + start_line: start, + end_line: end, + reason: format!("heuristic_body_stub:{:?}", mode), + }); + } else { + for (_, line) in collapsed.iter() { + let _ = writeln!(out, "{line}"); + } + } + collapsed.clear(); + }; + + for (idx, line) in content.lines().enumerate() { + let line_no = idx + 1; + let (opens, closes) = brace_delta(line); + let start_depth = depth; + let force_keep = KEEP_MARKERS.iter().any(|m| line.contains(m)); + + if start_depth == 0 || force_keep { + flush(&mut out, &mut collapsed, &mut elisions); + if let Some(symbol) = symbol_from_signature(line, line_no, line_no, "heuristic") { + symbols.push(symbol); + } + let _ = writeln!(out, "{line}"); + } else { + collapsed.push((line_no, line)); + } + depth += opens - closes; + if depth < 0 { + depth = 0; + } + } + flush(&mut out, &mut collapsed, &mut elisions); + + CodeStubOutput { + text: out.trim_end().to_string(), + symbols, + elisions, + parse_status: ParseStatus::HeuristicFallback, + } +} + +fn enforce_stub_budget(mut stub: CodeStubOutput, max_bytes: usize) -> CodeStubOutput { + if max_bytes == 0 || stub.text.len() <= max_bytes { + return stub; + } + + let marker = "\n[TOKENJUICE CODE STUB TRUNCATED]\n"; + let budget = max_bytes.saturating_sub(marker.len()).max(marker.len()); + let mut out = String::new(); + let mut kept_lines = 0usize; + for line in stub.text.lines() { + let addition = line.len() + 1; + if out.len() + addition + marker.len() > budget { + break; + } + out.push_str(line); + out.push('\n'); + kept_lines += 1; + } + out.push_str(marker); + let total_lines = stub.text.lines().count(); + if kept_lines < total_lines { + stub.elisions.push(CodeElision { + start_line: kept_lines.saturating_add(1), + end_line: total_lines, + reason: "stub_budget".to_string(), + }); + } + stub.text = out.trim_end().to_string(); + stub +} + +fn symbol_from_signature( + signature: &str, + start_line: usize, + end_line: usize, + fallback_kind: &str, +) -> Option { + let trimmed = signature.trim(); + if trimmed.is_empty() { + return None; + } + + let patterns = [ + ("fn ", "function"), + ("function ", "function"), + ("def ", "function"), + ("class ", "class"), + ("struct ", "struct"), + ("enum ", "enum"), + ("trait ", "trait"), + ("interface ", "interface"), + ("impl ", "impl"), + ("const ", "const"), + ("let ", "binding"), + ("var ", "binding"), + ]; + let lower = trimmed.to_ascii_lowercase(); + for (needle, kind) in patterns { + if let Some(idx) = lower.find(needle) { + let rest = &trimmed[idx + needle.len()..]; + let name = rest + .trim_start() + .trim_start_matches("r#") + .split(|c: char| !(c.is_ascii_alphanumeric() || c == '_' || c == '$')) + .next() + .unwrap_or_default(); + if !name.is_empty() { + return Some(SymbolSummary { + name: name.to_string(), + kind: kind.to_string(), + start_line, + end_line, + public: is_public_signature(trimmed), + }); + } + } + } + + if fallback_kind != "heuristic" { + Some(SymbolSummary { + name: fallback_kind.to_string(), + kind: fallback_kind.to_string(), + start_line, + end_line, + public: is_public_signature(trimmed), + }) + } else { + None + } +} + +fn is_public_signature(signature: &str) -> bool { + let trimmed = signature.trim_start(); + trimmed.starts_with("pub ") + || trimmed.starts_with("pub(") + || trimmed.starts_with("export ") + || trimmed.starts_with("public ") +} + +fn line_starts(content: &str) -> Vec { + let mut starts = vec![0usize]; + for (idx, byte) in content.bytes().enumerate() { + if byte == b'\n' { + starts.push(idx + 1); + } + } + starts +} + +fn byte_to_line(starts: &[usize], byte: usize) -> usize { + starts.partition_point(|start| *start <= byte).max(1) +} + /// Count `{`/`}` (and `(`/`)`) on a line, ignoring those inside string/char /// literals and line comments — a cheap approximation good enough for the /// depth heuristic. @@ -149,7 +440,11 @@ fn brace_delta(line: &str) -> (i32, i32) { /// imports, type declarations and struct/enum fields exactly. #[cfg(feature = "tokenjuice-treesitter")] mod treesitter { - use super::{CompressOutput, CompressorKind, MIN_BODY_LINES_TO_COLLAPSE}; + use super::{ + CompressOutput, CompressorKind, LineRange, MIN_BODY_LINES_TO_COLLAPSE, ParseStatus, + StubBody, StubMode, SymbolSummary, build_stub_from_bodies, byte_to_line, line_starts, + symbol_from_signature, + }; use tree_sitter::{Node, Parser}; /// Pick the grammar for a file extension. Returns the language plus whether @@ -241,6 +536,33 @@ mod treesitter { Some(CompressOutput::lossy(out, CompressorKind::Code)) } + pub fn stub(content: &str, ext: &str, mode: &StubMode) -> Option { + let (language, _braced) = language_for(ext)?; + let mut parser = Parser::new(); + parser.set_language(&language).ok()?; + let tree = parser.parse(content, None)?; + if tree.root_node().has_error() { + return None; + } + + let src = content.as_bytes(); + let starts = line_starts(content); + let mut bodies = Vec::new(); + collect_stub_bodies(tree.root_node(), content, src, &starts, &mut bodies); + bodies.sort_by_key(|body| body.body_start); + bodies.dedup_by_key(|body| (body.body_start, body.body_end)); + if bodies.is_empty() { + return None; + } + + Some(build_stub_from_bodies( + content, + bodies, + mode, + ParseStatus::TreeSitter, + )) + } + /// Recursively collect the byte-ranges of function/method bodies. fn collect_bodies(node: Node, src: &[u8], out: &mut Vec<(usize, usize)>) { if BODY_PARENTS.contains(&node.kind()) @@ -256,6 +578,51 @@ mod treesitter { collect_bodies(child, src, out); } } + + fn collect_stub_bodies( + node: Node, + content: &str, + src: &[u8], + starts: &[usize], + out: &mut Vec, + ) { + if BODY_PARENTS.contains(&node.kind()) + && let Some(body) = node.child_by_field_name("body") + { + let body_start = body.start_byte(); + let body_end = body.end_byte(); + let start_line = byte_to_line(starts, body_start); + let end_line = byte_to_line(starts, body_end.saturating_sub(1)); + let symbol = symbol_for_node(content, node, body, starts); + out.push(StubBody { + body_start, + body_end, + body_range: LineRange::new(start_line, end_line), + symbol, + }); + let _ = src; + return; + } + let mut cursor = node.walk(); + for child in node.children(&mut cursor) { + collect_stub_bodies(child, content, src, starts, out); + } + } + + fn symbol_for_node(content: &str, node: Node, body: Node, starts: &[usize]) -> SymbolSummary { + let signature = &content[node.start_byte()..body.start_byte()]; + let start_line = byte_to_line(starts, node.start_byte()); + let end_line = byte_to_line(starts, body.start_byte()); + symbol_from_signature(signature, start_line, end_line, node.kind()).unwrap_or_else(|| { + SymbolSummary { + name: node.kind().to_string(), + kind: node.kind().to_string(), + start_line, + end_line, + public: false, + } + }) + } } #[cfg(test)] @@ -354,4 +721,63 @@ mod tests { let out = compress_heuristic(&src).expect("compresses"); assert!(out.text.contains("TODO"), "marker line kept:\n{}", out.text); } + + #[test] + fn stub_code_reports_elisions_and_symbols() { + let mut src = String::from("use std::fmt;\n\npub fn visible() {\n"); + for i in 0..20 { + src.push_str(&format!(" println!(\"{i}\");\n")); + } + src.push_str("}\n"); + + let out = stub_code(&src, Some("rs"), &StubMode::SignaturesOnly, usize::MAX); + + assert_eq!(out.parse_status, ParseStatus::TreeSitter); + assert!(out.text.contains("pub fn visible()")); + assert!(out.text.contains("lines")); + assert!(!out.text.contains("println!(\"10\")")); + assert_eq!(out.symbols[0].name, "visible"); + assert!(out.symbols[0].public); + assert_eq!(out.elisions.len(), 1); + assert!(out.elisions[0].start_line <= out.elisions[0].end_line); + } + + #[test] + fn stub_code_expands_matched_symbol() { + let mut src = String::from("pub fn visible() {\n"); + for i in 0..10 { + src.push_str(&format!(" println!(\"visible {i}\");\n")); + } + src.push_str("}\n\nfn hidden() {\n"); + for i in 0..10 { + src.push_str(&format!(" println!(\"hidden {i}\");\n")); + } + src.push_str("}\n"); + + let out = stub_code( + &src, + Some("rs"), + &StubMode::MatchedSymbols(vec!["visible".to_string()]), + usize::MAX, + ); + + assert!(out.text.contains("visible 9"), "{}", out.text); + assert!(!out.text.contains("hidden 9"), "{}", out.text); + assert_eq!(out.elisions.len(), 1); + } + + #[test] + fn stub_code_reports_heuristic_fallback_for_unknown_language() { + let mut src = String::from("function f() {\n"); + for i in 0..12 { + src.push_str(&format!(" call({i});\n")); + } + src.push_str("}\n"); + + let out = stub_code(&src, Some("unknown"), &StubMode::SignaturesOnly, usize::MAX); + + assert_eq!(out.parse_status, ParseStatus::HeuristicFallback); + assert!(!out.elisions.is_empty()); + assert!(out.text.contains("lines")); + } } diff --git a/src/lib.rs b/src/lib.rs index ae51171..d72fc92 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -33,6 +33,7 @@ pub use compress::{ pub use compressor::{ CompressionInput, CompressionOutput, CompressionReport, Compressor, PassthroughCompressor, }; +pub use compressors::code::stub_code; pub use compressors::web_extract::{ reduce_web_extract, reduce_web_extract_batch_with_store, reduce_web_extract_with_store, replace_inline_base64_images, @@ -61,8 +62,9 @@ pub use tool_integration::{ configure, current_options, install_config, }; pub use types::{ - AgentTokenjuiceCompression, CompactResult, CompressInput, CompressOptions, CompressOutput, - CompressedOutput, CompressorKind, ContentHint, ContentKind, ReduceOptions, ToolExecutionInput, + AgentTokenjuiceCompression, CodeElision, CodeStubOutput, CompactResult, CompressInput, + CompressOptions, CompressOutput, CompressedOutput, CompressorKind, ContentHint, ContentKind, + LineRange, ParseStatus, ReadIntent, ReduceOptions, StubMode, SymbolSummary, ToolExecutionInput, WebExtractBatchInput, WebExtractFormat, WebExtractOptions, WebExtractReduceInput, WebExtractReduction, }; diff --git a/src/types.rs b/src/types.rs index 3d7b10c..1782f76 100644 --- a/src/types.rs +++ b/src/types.rs @@ -393,6 +393,89 @@ pub struct ContentHint { pub query: Option, /// Hard override — when set, detection returns this kind verbatim. pub explicit: Option, + /// Whether file-read content must remain exact or may be reduced to a code + /// stub. Defaults to exact; hosts must opt into stubbing explicitly. + pub read_intent: ReadIntent, +} + +/// Caller intent for read-like sources. +#[derive(Debug, Clone, PartialEq, Eq, Default)] +pub enum ReadIntent { + /// Preserve bytes exactly. This is the default for file reads. + #[default] + Exact, + /// Return a structural source-code stub. + Stub(StubMode), +} + +/// Source-code stub mode. +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", tag = "kind", content = "value")] +pub enum StubMode { + #[default] + SignaturesOnly, + PublicApi, + MatchedSymbols(Vec), + ExpandAroundLines(Vec), +} + +/// One-based inclusive line range. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct LineRange { + pub start: usize, + pub end: usize, +} + +impl LineRange { + pub fn new(start: usize, end: usize) -> Self { + Self { + start: start.max(1), + end: end.max(start.max(1)), + } + } + + pub fn intersects(self, other: Self) -> bool { + self.start <= other.end && other.start <= self.end + } +} + +/// Parser path used for a source-code stub. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum ParseStatus { + TreeSitter, + HeuristicFallback, +} + +/// Symbol surfaced in a source-code stub. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct SymbolSummary { + pub name: String, + pub kind: String, + pub start_line: usize, + pub end_line: usize, + pub public: bool, +} + +/// Source range omitted from a stub. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct CodeElision { + pub start_line: usize, + pub end_line: usize, + pub reason: String, +} + +/// Structured source-code stub result. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct CodeStubOutput { + pub text: String, + pub symbols: Vec, + pub elisions: Vec, + pub parse_status: ParseStatus, } // --------------------------------------------------------------------------- diff --git a/tests/e2e_tool_output.rs b/tests/e2e_tool_output.rs index 3ef9bfe..447cb4e 100644 --- a/tests/e2e_tool_output.rs +++ b/tests/e2e_tool_output.rs @@ -3,8 +3,8 @@ //! These exercise the same path OpenHuman's `ToolOutputMiddleware` uses: //! `compact_tool_output_with_policy` → router → compressor → CCR offload → //! recovery footer → `cache::retrieve` roundtrip. They pin the plan-level -//! acceptance criteria: `off` and recovery-tool output are exact passthrough, -//! `light` declines lossy compaction, `full` compacts recoverably. +//! acceptance criteria: `off`, recovery-tool output, and exact file reads are +//! passthrough; `light` declines lossy compaction; `full` compacts recoverably. mod common; @@ -95,8 +95,8 @@ async fn full_profile_compacts_and_original_is_recoverable() { common::install_test_config(); let payload = big_json_rows(); let (text, stats) = compact_tool_output_with_policy( - "read_file", - Some(&json!({ "path": "services.json" })), + "http_request", + Some(&json!({ "url": "https://example.test/services" })), &payload, Some(0), AgentTokenjuiceCompression::Full, @@ -121,6 +121,23 @@ async fn full_profile_compacts_and_original_is_recoverable() { assert_eq!(original, payload, "CCR roundtrip must restore the original"); } +#[tokio::test] +async fn file_read_is_exact_by_default_even_in_full_profile() { + common::install_test_config(); + let payload = big_json_rows(); + let (text, stats) = compact_tool_output_with_policy( + "read_file", + Some(&json!({ "path": "services.json" })), + &payload, + Some(0), + AgentTokenjuiceCompression::Full, + ) + .await; + + assert_eq!(text, payload, "default file reads must remain exact"); + assert!(!stats.applied); +} + #[tokio::test] async fn exit_code_and_command_reach_the_rule_engine() { common::install_test_config(); From 6319d31fbe1d3dc4ed5778186809aeddd601c4e5 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Sun, 5 Jul 2026 08:23:36 +0000 Subject: [PATCH 005/120] Add richer bloat estimators --- src/pipeline/estimate.rs | 360 ++++++++++++++++++++++++++++++++++++--- 1 file changed, 337 insertions(+), 23 deletions(-) diff --git a/src/pipeline/estimate.rs b/src/pipeline/estimate.rs index c680f49..ab8b2c7 100644 --- a/src/pipeline/estimate.rs +++ b/src/pipeline/estimate.rs @@ -13,9 +13,13 @@ pub struct BloatEstimate { #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum BloatReason { JsonRows, + JsonSaturation, DiffContext, + DiffNoise, LogRepetition, + LogTemplates, SearchFanout, + SearchClustering, HtmlMarkup, CodeBody, TextRepetition, @@ -26,9 +30,13 @@ impl BloatReason { pub fn as_str(self) -> &'static str { match self { Self::JsonRows => "json_rows", + Self::JsonSaturation => "json_saturation", Self::DiffContext => "diff_context", + Self::DiffNoise => "diff_noise", Self::LogRepetition => "log_repetition", + Self::LogTemplates => "log_templates", Self::SearchFanout => "search_fanout", + Self::SearchClustering => "search_clustering", Self::HtmlMarkup => "html_markup", Self::CodeBody => "code_body", Self::TextRepetition => "text_repetition", @@ -68,36 +76,125 @@ fn estimate_json_bloat(content: &str) -> BloatEstimate { return low(); } let object_rows = rows.iter().filter(|row| row.is_object()).count(); - let score = (40 + score_ratio(object_rows.saturating_sub(2), rows.len()) / 2).min(95); - BloatEstimate { - score, - reason: BloatReason::JsonRows, + if object_rows == 0 { + return low(); } + + let mut columns = std::collections::BTreeSet::new(); + let mut row_shapes = std::collections::HashSet::new(); + let mut row_hashes = std::collections::HashSet::new(); + let mut scalar_values_by_key: std::collections::HashMap< + String, + std::collections::HashSet, + > = std::collections::HashMap::new(); + let mut present_cells = 0usize; + + for row in rows.iter().filter_map(|row| row.as_object()) { + let mut shape = Vec::with_capacity(row.len()); + for (key, value) in row { + columns.insert(key.clone()); + shape.push(key.as_str()); + present_cells += 1; + if value.is_string() || value.is_boolean() || value.is_number() || value.is_null() { + scalar_values_by_key + .entry(key.clone()) + .or_default() + .insert(stable_value_hash(value)); + } + } + shape.sort_unstable(); + row_shapes.insert(shape.join("\0")); + row_hashes.insert(stable_object_hash(row)); + } + + let total_cells = object_rows.saturating_mul(columns.len()).max(1); + let sparse_cells = total_cells.saturating_sub(present_cells); + let constant_fields = scalar_values_by_key + .values() + .filter(|values| values.len() == 1) + .count(); + let duplicate_rows = object_rows.saturating_sub(row_hashes.len()); + let saturation_score = (score_ratio(duplicate_rows, object_rows) / 3) + + (score_ratio(constant_fields, columns.len().max(1)) / 3) + + (score_ratio(sparse_cells, total_cells) / 3); + let row_pressure = score_ratio(object_rows.saturating_sub(2), rows.len()) / 2; + let shape_pressure = score_ratio(row_shapes.len().saturating_sub(1), object_rows.max(1)) / 4; + let score = (40 + row_pressure + saturation_score + shape_pressure).min(95); + let reason = if saturation_score >= 20 || duplicate_rows >= object_rows / 4 { + BloatReason::JsonSaturation + } else { + BloatReason::JsonRows + }; + BloatEstimate { score, reason } } fn estimate_diff_bloat(content: &str) -> BloatEstimate { - let mut context = 0usize; - let mut changed = 0usize; - for line in content.lines().take(2_000) { - if line.starts_with("@@") - || line.starts_with("diff --git") - || line.starts_with("+++") - || line.starts_with("---") - { + let mut context_bytes = 0usize; + let mut changed_bytes = 0usize; + let mut noisy_bytes = 0usize; + let mut whitespace_only_bytes = 0usize; + let mut current_file_is_noisy = false; + let mut hunk_lines: Vec<&str> = Vec::new(); + + let flush_hunk = |hunk_lines: &mut Vec<&str>, whitespace_only_bytes: &mut usize| { + if hunk_is_whitespace_only(hunk_lines) { + *whitespace_only_bytes += hunk_lines + .iter() + .filter(|line| is_changed_diff_line(line)) + .map(|line| line.len() + 1) + .sum::(); + } + hunk_lines.clear(); + }; + + for line in content.lines().take(4_000) { + if line.starts_with("diff --git") { + flush_hunk(&mut hunk_lines, &mut whitespace_only_bytes); + current_file_is_noisy = diff_path_is_noisy(line); continue; } - if line.starts_with('+') || line.starts_with('-') { - changed += 1; - } else if line.starts_with(' ') || !line.trim().is_empty() { - context += 1; + if line.starts_with("@@") { + flush_hunk(&mut hunk_lines, &mut whitespace_only_bytes); + continue; + } + if line.starts_with("+++") || line.starts_with("---") { + continue; + } + + if is_changed_diff_line(line) { + let bytes = line.len() + 1; + changed_bytes += bytes; + hunk_lines.push(line); + if current_file_is_noisy { + noisy_bytes += bytes; + } + } else if line.starts_with(' ') { + let bytes = line.len() + 1; + context_bytes += bytes; + hunk_lines.push(line); + if current_file_is_noisy { + noisy_bytes += bytes; + } + } else if !line.trim().is_empty() && current_file_is_noisy { + noisy_bytes += line.len() + 1; } } - let total = context + changed; + flush_hunk(&mut hunk_lines, &mut whitespace_only_bytes); + + let total = context_bytes + changed_bytes; if total == 0 { return low(); } + let droppable_noise = (noisy_bytes + whitespace_only_bytes).min(total); + if droppable_noise > 0 { + let score = score_ratio(droppable_noise, total).max(score_ratio(context_bytes, total) / 2); + return BloatEstimate { + score, + reason: BloatReason::DiffNoise, + }; + } BloatEstimate { - score: score_ratio(context, total), + score: score_ratio(context_bytes, total), reason: BloatReason::DiffContext, } } @@ -105,30 +202,52 @@ fn estimate_diff_bloat(content: &str) -> BloatEstimate { fn estimate_log_bloat(content: &str) -> BloatEstimate { let mut total = 0usize; let mut repeated = 0usize; + let mut priority = 0usize; let mut previous = ""; - for line in content.lines().take(2_000) { + let mut templates: std::collections::HashMap = std::collections::HashMap::new(); + for line in content.lines().take(4_000) { let trimmed = line.trim(); if trimmed.is_empty() { continue; } total += 1; + if log_line_has_priority(trimmed) { + priority += 1; + } if trimmed == previous { repeated += 1; } previous = trimmed; + let template = log_template(trimmed); + if template != trimmed { + *templates.entry(template).or_default() += 1; + } } if total == 0 { return low(); } + let template_repetition = templates + .values() + .filter(|count| **count >= 3) + .map(|count| count.saturating_sub(1)) + .sum::(); + let priority_dilution = total.saturating_sub(priority).saturating_sub(80); + let score = score_ratio(repeated + template_repetition + priority_dilution, total).min(95); + if template_repetition >= 10 { + return BloatEstimate { + score, + reason: BloatReason::LogTemplates, + }; + } BloatEstimate { - score: score_ratio(repeated + total.saturating_sub(80), total), + score, reason: BloatReason::LogRepetition, } } fn estimate_search_bloat(content: &str) -> BloatEstimate { let mut total = 0usize; - let mut paths = std::collections::HashSet::new(); + let mut paths: std::collections::HashMap<&str, usize> = std::collections::HashMap::new(); for line in content.lines().take(2_000) { let Some((path, _rest)) = line.split_once(':') else { continue; @@ -137,12 +256,19 @@ fn estimate_search_bloat(content: &str) -> BloatEstimate { continue; } total += 1; - paths.insert(path); + *paths.entry(path).or_default() += 1; } if total == 0 { return low(); } let fanout = paths.len(); + let max_cluster = paths.values().copied().max().unwrap_or(0); + if total >= 20 && score_ratio(max_cluster, total) >= 60 { + return BloatEstimate { + score: score_ratio(max_cluster, total).min(95), + reason: BloatReason::SearchClustering, + }; + } BloatEstimate { score: (score_ratio(total.saturating_sub(20), total) / 2 + score_ratio(fanout.saturating_sub(3), fanout.max(1)) / 2) @@ -214,6 +340,109 @@ fn low() -> BloatEstimate { } } +fn stable_value_hash(value: &serde_json::Value) -> u64 { + stable_hash(serde_json::to_string(value).unwrap_or_default().as_bytes()) +} + +fn stable_object_hash(object: &serde_json::Map) -> u64 { + let mut bytes = Vec::new(); + for (key, value) in object { + bytes.extend_from_slice(key.as_bytes()); + bytes.push(0); + bytes.extend_from_slice(serde_json::to_string(value).unwrap_or_default().as_bytes()); + bytes.push(0xff); + } + stable_hash(&bytes) +} + +fn stable_hash(bytes: &[u8]) -> u64 { + let mut hash = 0xcbf29ce484222325u64; + for byte in bytes { + hash ^= u64::from(*byte); + hash = hash.wrapping_mul(0x100000001b3); + } + hash +} + +fn is_changed_diff_line(line: &str) -> bool { + (line.starts_with('+') && !line.starts_with("+++")) + || (line.starts_with('-') && !line.starts_with("---")) +} + +fn hunk_is_whitespace_only(lines: &[&str]) -> bool { + let removed: Vec = lines + .iter() + .filter_map(|line| line.strip_prefix('-')) + .map(strip_ascii_whitespace) + .collect(); + let added: Vec = lines + .iter() + .filter_map(|line| line.strip_prefix('+')) + .map(strip_ascii_whitespace) + .collect(); + !removed.is_empty() && removed.len() == added.len() && removed == added +} + +fn strip_ascii_whitespace(s: &str) -> String { + s.chars().filter(|c| !c.is_ascii_whitespace()).collect() +} + +fn diff_path_is_noisy(diff_git_line: &str) -> bool { + let line = diff_git_line.to_ascii_lowercase(); + const NOISY_SUFFIXES: &[&str] = &[ + "cargo.lock", + "package-lock.json", + "pnpm-lock.yaml", + "yarn.lock", + "composer.lock", + "poetry.lock", + "gemfile.lock", + "go.sum", + ".min.js", + ".min.css", + ".map", + ]; + NOISY_SUFFIXES.iter().any(|suffix| line.contains(suffix)) +} + +fn log_line_has_priority(line: &str) -> bool { + const PRIORITY_MARKERS: &[&str] = &[ + "error", + "failed", + "failure", + "panic", + "exception", + "warning", + "warn", + "fatal", + "traceback", + ]; + let lower = line.to_ascii_lowercase(); + PRIORITY_MARKERS.iter().any(|marker| lower.contains(marker)) +} + +fn log_template(line: &str) -> String { + let mut out = String::with_capacity(line.len()); + let mut chars = line.chars().peekable(); + while let Some(ch) = chars.next() { + if ch == '0' && chars.peek() == Some(&'x') { + out.push_str("0x#"); + chars.next(); + while chars.peek().is_some_and(|next| next.is_ascii_hexdigit()) { + chars.next(); + } + } else if ch.is_ascii_digit() { + out.push('#'); + while chars.peek().is_some_and(|next| next.is_ascii_digit()) { + chars.next(); + } + } else { + out.push(ch); + } + } + out +} + #[cfg(test)] mod tests { use super::*; @@ -222,11 +451,30 @@ mod tests { fn json_rows_estimate_is_redacted() { let content = r#"[{"secret":"a"},{"secret":"b"},{"secret":"c"}]"#; let estimate = estimate_bloat(content, ContentKind::Json); - assert_eq!(estimate.reason, BloatReason::JsonRows); + assert!(matches!( + estimate.reason, + BloatReason::JsonRows | BloatReason::JsonSaturation + )); assert!(estimate.score > 0); assert!(!estimate.reason.as_str().contains("secret")); } + #[test] + fn json_saturation_detects_duplicate_constant_rows() { + let mut rows = Vec::new(); + for i in 0..80 { + rows.push(format!( + r#"{{"id":{},"status":"ok","region":"us","shape":"same"}}"#, + i % 4 + )); + } + let content = format!("[{}]", rows.join(",")); + let estimate = estimate_bloat(&content, ContentKind::Json); + + assert_eq!(estimate.reason, BloatReason::JsonSaturation); + assert!(estimate.score >= 60, "{estimate:?}"); + } + #[test] fn diff_context_estimate_detects_context_heavy_payload() { let mut diff = String::from("diff --git a/a b/a\n@@ -1,8 +1,8 @@\n"); @@ -239,6 +487,72 @@ mod tests { assert!(estimate.score > 80); } + #[test] + fn diff_noise_estimate_detects_lockfile_hunks() { + let mut diff = String::from("diff --git a/Cargo.lock b/Cargo.lock\n"); + diff.push_str("@@ -1,20 +1,40 @@\n"); + for i in 0..40 { + diff.push_str(&format!("+ dep version {i}\n")); + } + for i in 0..20 { + diff.push_str(&format!("- old dep version {i}\n")); + } + + let estimate = estimate_bloat(&diff, ContentKind::Diff); + + assert_eq!(estimate.reason, BloatReason::DiffNoise); + assert!(estimate.score > 80, "{estimate:?}"); + } + + #[test] + fn diff_noise_estimate_detects_whitespace_only_hunks() { + let diff = "\ +diff --git a/src/lib.rs b/src/lib.rs +@@ -1,2 +1,2 @@ +-fn main(){println!(\"hi\");} ++fn main() { println!(\"hi\"); } + context +"; + + let estimate = estimate_bloat(diff, ContentKind::Diff); + + assert_eq!(estimate.reason, BloatReason::DiffNoise); + assert!(estimate.score > 0, "{estimate:?}"); + } + + #[test] + fn log_template_estimate_detects_repeated_variants() { + let mut log = String::new(); + for i in 0..80 { + log.push_str(&format!( + "2026-07-05T12:{:02}:00Z worker-{i} processed item id={}\n", + i % 60, + 10_000 + i + )); + } + + let estimate = estimate_bloat(&log, ContentKind::Log); + + assert_eq!(estimate.reason, BloatReason::LogTemplates); + assert!(estimate.score > 50, "{estimate:?}"); + } + + #[test] + fn search_clustering_estimate_detects_single_file_fan_in() { + let mut output = String::new(); + for i in 0..40 { + output.push_str(&format!("src/lib.rs:{i}:needle match {i}\n")); + } + for i in 0..5 { + output.push_str(&format!("src/other.rs:{i}:needle match {i}\n")); + } + + let estimate = estimate_bloat(&output, ContentKind::Search); + + assert_eq!(estimate.reason, BloatReason::SearchClustering); + assert!(estimate.score >= 80, "{estimate:?}"); + } + #[test] fn plain_unique_text_is_low_signal() { let estimate = estimate_bloat("alpha\nbeta\ngamma", ContentKind::PlainText); From b52ec8995cd3175640de28e1e22f06c7a986bd72 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Sun, 5 Jul 2026 08:36:53 +0000 Subject: [PATCH 006/120] Add DiffNoise and log template reformat --- src/compress.rs | 30 +++++ src/compressors/diff.rs | 222 +++++++++++++++++++++++++++------ src/compressors/log.rs | 264 +++++++++++++++++++++++++++++++++++++++- 3 files changed, 480 insertions(+), 36 deletions(-) diff --git a/src/compress.rs b/src/compress.rs index 9afdfe2..e3678d0 100644 --- a/src/compress.rs +++ b/src/compress.rs @@ -360,6 +360,36 @@ mod tests { ); } + #[tokio::test] + async fn diff_noise_output_offloads_original_for_recovery() { + let mut original = String::from("diff --git a/Cargo.lock b/Cargo.lock\n"); + original.push_str("@@ -1,800 +1,900 @@\n"); + for i in 0..700 { + original.push_str(&format!("+ new dependency entry {i} checksum {}\n", i * 17)); + } + for i in 0..350 { + original.push_str(&format!("- old dependency entry {i} checksum {}\n", i * 19)); + } + let hint = ContentHint { + explicit: Some(ContentKind::Diff), + ..Default::default() + }; + let store = cache::MemoryCcrStore::new(10, 1_000_000); + + let res = compress_content_with_store(&original, Some(hint), &opts(), &store).await; + + assert!(res.applied); + assert!(res.lossy); + assert_eq!(res.content_kind, ContentKind::Diff); + assert!( + res.text.contains("reason=lockfile_or_bundle"), + "{}", + res.text + ); + let token = res.ccr_token.as_deref().expect("offloaded"); + assert_eq!(store.get(token).as_deref(), Some(original.as_str())); + } + #[tokio::test] async fn exposes_body_and_recovery_footer_separately() { let mut rows = Vec::new(); diff --git a/src/compressors/diff.rs b/src/compressors/diff.rs index ee28cf3..34d8968 100644 --- a/src/compressors/diff.rs +++ b/src/compressors/diff.rs @@ -18,6 +18,43 @@ pub const CONTEXT_ANCHOR: usize = 3; /// A run of unchanged context longer than this collapses to a marker. pub const CONTEXT_COLLAPSE_THRESHOLD: usize = 8; +const DEFAULT_NOISY_PATH_SUBSTRINGS: &[&str] = &[ + "cargo.lock", + "package-lock.json", + "pnpm-lock.yaml", + "yarn.lock", + "composer.lock", + "poetry.lock", + "gemfile.lock", + ".min.js", + ".min.css", + ".map", + "go.sum", +]; + +/// Controls the low-value diff hunk offload policy. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct DiffNoiseOptions { + /// Case-insensitive path substrings that identify generated or lockfile + /// hunks whose body can be represented by a reason marker. + pub noisy_path_substrings: Vec, + /// Drop hunks whose added/removed lines are identical after ASCII + /// whitespace stripping. Off by default until fixture coverage is broad. + pub drop_whitespace_only_hunks: bool, +} + +impl Default for DiffNoiseOptions { + fn default() -> Self { + Self { + noisy_path_substrings: DEFAULT_NOISY_PATH_SUBSTRINGS + .iter() + .map(|value| value.to_string()) + .collect(), + drop_whitespace_only_hunks: false, + } + } +} + pub struct DiffCompressor; #[async_trait] @@ -38,6 +75,14 @@ impl Compressor for DiffCompressor { /// Compress a unified diff. Returns `None` when there's nothing structural to /// work with or compression wouldn't shrink it. pub fn compress(content: &str) -> Option { + compress_with_options(content, &DiffNoiseOptions::default()) +} + +/// Compress a unified diff with explicit diff-noise policy. +pub fn compress_with_options( + content: &str, + noise_options: &DiffNoiseOptions, +) -> Option { let stripped = strip_ansi(content); let lines: Vec<&str> = stripped.lines().collect(); if lines.is_empty() { @@ -53,22 +98,38 @@ pub fn compress(content: &str) -> Option { let line = lines[i]; if line.starts_with("diff --git ") { - current_file_is_noisy = is_noisy_path(line); + current_file_is_noisy = is_noisy_path(line, noise_options); let _ = writeln!(out, "{line}"); i += 1; continue; } if is_structural(line) { saw_hunk |= line.starts_with("@@"); - if current_file_is_noisy && line.starts_with("@@") { + if line.starts_with("@@") { let _ = writeln!(out, "{line}"); + let hunk_start = i + 1; + let hunk_end = hunk_body_end(&lines, hunk_start); + let hunk_body = &lines[hunk_start..hunk_end]; + let reason = if current_file_is_noisy { + Some("lockfile_or_bundle") + } else if noise_options.drop_whitespace_only_hunks + && hunk_is_whitespace_only(hunk_body) + { + Some("whitespace_only") + } else { + None + }; + if let Some(reason) = reason { + let summary = summarize_hunk_body(hunk_body); + let _ = writeln!( + out, + "[... diff-noise hunk omitted reason={reason} +{}/-{} context={} line(s) ...]", + summary.added, summary.removed, summary.context + ); + i = hunk_end; + continue; + } i += 1; - let (added, removed, consumed) = summarize_hunk_body(&lines[i..]); - let _ = writeln!( - out, - "[... lockfile/bundle hunk: +{added}/-{removed} line(s) omitted ...]" - ); - i += consumed; continue; } let _ = writeln!(out, "{line}"); @@ -137,40 +198,59 @@ fn is_context(line: &str) -> bool { line.starts_with(' ') } -fn summarize_hunk_body(lines: &[&str]) -> (usize, usize, usize) { - let mut added = 0usize; - let mut removed = 0usize; - let mut n = 0usize; +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +struct HunkSummary { + added: usize, + removed: usize, + context: usize, +} + +fn hunk_body_end(lines: &[&str], start: usize) -> usize { + let mut i = start; + while i < lines.len() && !lines[i].starts_with("@@") && !lines[i].starts_with("diff --git ") { + i += 1; + } + i +} + +fn summarize_hunk_body(lines: &[&str]) -> HunkSummary { + let mut summary = HunkSummary::default(); for &line in lines { - if line.starts_with("@@") || line.starts_with("diff --git ") { - break; - } if line.starts_with('+') && !line.starts_with("+++") { - added += 1; + summary.added += 1; } else if line.starts_with('-') && !line.starts_with("---") { - removed += 1; + summary.removed += 1; + } else if line.starts_with(' ') { + summary.context += 1; } - n += 1; } - (added, removed, n) + summary } -fn is_noisy_path(diff_git_line: &str) -> bool { +fn hunk_is_whitespace_only(lines: &[&str]) -> bool { + let removed: Vec = lines + .iter() + .filter_map(|line| line.strip_prefix('-')) + .map(strip_ascii_whitespace) + .collect(); + let added: Vec = lines + .iter() + .filter_map(|line| line.strip_prefix('+')) + .map(strip_ascii_whitespace) + .collect(); + !removed.is_empty() && removed.len() == added.len() && removed == added +} + +fn strip_ascii_whitespace(s: &str) -> String { + s.chars().filter(|c| !c.is_ascii_whitespace()).collect() +} + +fn is_noisy_path(diff_git_line: &str, options: &DiffNoiseOptions) -> bool { let l = diff_git_line.to_ascii_lowercase(); - const NOISY: &[&str] = &[ - "cargo.lock", - "package-lock.json", - "pnpm-lock.yaml", - "yarn.lock", - "composer.lock", - "poetry.lock", - "gemfile.lock", - ".min.js", - ".min.css", - ".map", - "go.sum", - ]; - NOISY.iter().any(|p| l.contains(p)) + options.noisy_path_substrings.iter().any(|p| { + let p = p.to_ascii_lowercase(); + !p.is_empty() && l.contains(&p) + }) } #[cfg(test)] @@ -206,11 +286,83 @@ mod tests { let _ = writeln!(s, "- old dep entry {i}"); } let out = compress(&s).expect("compresses").text; - assert!(out.contains("lockfile/bundle hunk"), "{out}"); + assert!( + out.contains("diff-noise hunk omitted reason=lockfile_or_bundle"), + "{out}" + ); assert!(!out.contains("new dep entry 7"), "{out}"); assert!(out.len() < s.len()); } + #[test] + fn whitespace_only_hunks_are_preserved_by_default() { + let mut s = String::from("diff --git a/src/lib.rs b/src/lib.rs\n@@ -1,40 +1,40 @@\n"); + for i in 0..20 { + let _ = writeln!(s, " context before {i}"); + } + let _ = writeln!(s, "-fn main(){{println!(\"hi\");}}"); + let _ = writeln!(s, "+fn main() {{ println!(\"hi\"); }}"); + for i in 0..20 { + let _ = writeln!(s, " context after {i}"); + } + let out = compress(&s).expect("compresses").text; + + assert!(out.contains("-fn main()"), "{out}"); + assert!(out.contains("+fn main()"), "{out}"); + assert!(!out.contains("reason=whitespace_only"), "{out}"); + } + + #[test] + fn whitespace_only_hunks_drop_when_enabled() { + let mut s = String::from("diff --git a/src/lib.rs b/src/lib.rs\n@@ -1,60 +1,60 @@\n"); + for i in 0..50 { + let _ = writeln!(s, " context before {i}"); + } + let _ = writeln!(s, "-fn main(){{println!(\"hi\");}}"); + let _ = writeln!(s, "+fn main() {{ println!(\"hi\"); }}"); + for i in 0..50 { + let _ = writeln!(s, " context after {i}"); + } + let options = DiffNoiseOptions { + drop_whitespace_only_hunks: true, + ..Default::default() + }; + let out = compress_with_options(&s, &options) + .expect("compresses") + .text; + + assert!( + out.contains("diff-noise hunk omitted reason=whitespace_only"), + "{out}" + ); + assert!(!out.contains("-fn main()"), "{out}"); + assert!(!out.contains("+fn main()"), "{out}"); + } + + #[test] + fn semantic_hunks_survive_whitespace_noise_mode() { + let mut s = String::from("diff --git a/src/lib.rs b/src/lib.rs\n@@ -1,40 +1,40 @@\n"); + for i in 0..20 { + let _ = writeln!(s, " context before {i}"); + } + let _ = writeln!(s, "-fn answer() -> i32 {{ 41 }}"); + let _ = writeln!(s, "+fn answer() -> i32 {{ 42 }}"); + for i in 0..20 { + let _ = writeln!(s, " context after {i}"); + } + let options = DiffNoiseOptions { + drop_whitespace_only_hunks: true, + ..Default::default() + }; + let out = compress_with_options(&s, &options) + .expect("compresses") + .text; + + assert!(out.contains("41"), "{out}"); + assert!(out.contains("42"), "{out}"); + assert!(!out.contains("diff-noise hunk omitted"), "{out}"); + } + #[test] fn non_diff_returns_none() { assert!(compress("just some text\nno hunks here").is_none()); diff --git a/src/compressors/log.rs b/src/compressors/log.rs index fe94821..8807032 100644 --- a/src/compressors/log.rs +++ b/src/compressors/log.rs @@ -33,6 +33,7 @@ pub const MAX_WARNINGS: usize = 5; pub const MAX_STACK_TRACES: usize = 3; pub const STACK_TRACE_MAX_LINES: usize = 20; pub const MAX_TOTAL_LINES: usize = 100; +pub const TEMPLATE_MIN_RUN: usize = 8; static BUILTIN_RULES: Lazy> = Lazy::new(load_builtin_rules); @@ -54,7 +55,7 @@ impl Compressor for LogCompressor { if has_command { compress_command(input, opts) } else { - compress_signal(input.content) + compress_templates(input.content).or_else(|| compress_signal(input.content)) } } } @@ -224,6 +225,145 @@ pub fn compress_signal(content: &str) -> Option { )) } +/// Lossless reformat for repetitive non-command logs. Consecutive runs of the +/// same line template are represented as one template plus captured variants. +pub fn compress_templates(content: &str) -> Option { + let lines: Vec<&str> = content.lines().collect(); + if lines.len() < TEMPLATE_MIN_RUN { + return None; + } + + let mut out = String::with_capacity(content.len() / 2 + 128); + let mut i = 0usize; + let mut emitted_block = false; + + while i < lines.len() { + let Some(first) = line_template(lines[i]) else { + let _ = writeln!(out, "{}", lines[i]); + i += 1; + continue; + }; + + let start = i; + let mut captures = vec![first.captures]; + i += 1; + while i < lines.len() { + let Some(next) = line_template(lines[i]) else { + break; + }; + if next.template != first.template || next.captures.len() != captures[0].len() { + break; + } + captures.push(next.captures); + i += 1; + } + + if captures.len() >= TEMPLATE_MIN_RUN { + write_template_block(&mut out, &first.template, &captures); + emitted_block = true; + } else { + for line in &lines[start..i] { + let _ = writeln!(out, "{line}"); + } + } + } + + let out = out.trim_end().to_string(); + if !emitted_block || out.len() >= content.len() { + return None; + } + log::debug!( + "[tokenjuice][log] template reformat {} -> {} bytes", + content.len(), + out.len(), + ); + Some(CompressOutput::reformatted(out, CompressorKind::Log)) +} + +#[derive(Debug, Clone, PartialEq, Eq)] +struct LineTemplate { + template: String, + captures: Vec, +} + +fn line_template(line: &str) -> Option { + let mut template = String::with_capacity(line.len()); + let mut captures = Vec::new(); + let mut chars = line.char_indices().peekable(); + + while let Some((idx, ch)) = chars.next() { + if ch == '0' && chars.peek().is_some_and(|(_, next)| *next == 'x') { + let start = idx; + let mut end = idx + ch.len_utf8(); + if let Some((x_idx, x_ch)) = chars.next() { + end = x_idx + x_ch.len_utf8(); + } + while let Some((next_idx, next_ch)) = chars.peek().copied() { + if next_ch.is_ascii_hexdigit() { + chars.next(); + end = next_idx + next_ch.len_utf8(); + } else { + break; + } + } + template.push_str("{}"); + captures.push(line[start..end].to_string()); + } else if ch.is_ascii_digit() { + let start = idx; + let mut end = idx + ch.len_utf8(); + while let Some((next_idx, next_ch)) = chars.peek().copied() { + if next_ch.is_ascii_digit() { + chars.next(); + end = next_idx + next_ch.len_utf8(); + } else { + break; + } + } + template.push_str("{}"); + captures.push(line[start..end].to_string()); + } else { + template.push(ch); + } + } + + if captures.len() < 2 || template.trim().len() < 12 { + return None; + } + Some(LineTemplate { template, captures }) +} + +fn write_template_block(out: &mut String, template: &str, captures: &[Vec]) { + let fields = captures.first().map_or(0, Vec::len); + let _ = writeln!( + out, + "[TOKENJUICE LOG TEMPLATE run={} fields={fields}]", + captures.len() + ); + let _ = writeln!(out, "{template}"); + for row in captures { + let rendered = row + .iter() + .map(|capture| escape_capture(capture)) + .collect::>() + .join("\t"); + let _ = writeln!(out, "{rendered}"); + } + let _ = writeln!(out, "[/TOKENJUICE LOG TEMPLATE]"); +} + +fn escape_capture(value: &str) -> String { + let mut out = String::with_capacity(value.len()); + for ch in value.chars() { + match ch { + '\\' => out.push_str("\\\\"), + '\t' => out.push_str("\\t"), + '\r' => out.push_str("\\r"), + _ => out.push(ch), + } + } + out +} + fn select_first_last(idx: &[usize], cap: usize) -> Vec { if idx.len() <= cap { return idx.to_vec(); @@ -285,6 +425,7 @@ fn normalize_for_dedupe(line: &str) -> String { #[cfg(test)] mod tests { use super::*; + use crate::types::{CompressOptions, ContentHint, ContentKind}; fn noisy_log() -> String { let mut s = String::new(); @@ -298,6 +439,127 @@ mod tests { s } + fn repetitive_template_log() -> String { + let mut s = String::new(); + for i in 0..80 { + let _ = writeln!( + s, + "2026-07-05T12:{:02}:00Z worker-{i} processed item id={} shard={}", + i % 60, + 10_000 + i, + i % 8 + ); + } + s + } + + fn reconstruct_template_reformat(output: &str) -> String { + let mut out = String::new(); + let lines: Vec<&str> = output.lines().collect(); + let mut i = 0usize; + while i < lines.len() { + if lines[i].starts_with("[TOKENJUICE LOG TEMPLATE ") { + let template = lines[i + 1]; + i += 2; + while i < lines.len() && lines[i] != "[/TOKENJUICE LOG TEMPLATE]" { + let captures = lines[i] + .split('\t') + .map(unescape_capture_for_test) + .collect::>(); + let _ = writeln!(out, "{}", apply_template_for_test(template, &captures)); + i += 1; + } + i += 1; + } else { + let _ = writeln!(out, "{}", lines[i]); + i += 1; + } + } + out + } + + fn unescape_capture_for_test(value: &str) -> String { + let mut out = String::new(); + let mut chars = value.chars(); + while let Some(ch) = chars.next() { + if ch == '\\' { + match chars.next() { + Some('\\') => out.push('\\'), + Some('t') => out.push('\t'), + Some('r') => out.push('\r'), + Some(other) => { + out.push('\\'); + out.push(other); + } + None => out.push('\\'), + } + } else { + out.push(ch); + } + } + out + } + + fn apply_template_for_test(template: &str, captures: &[String]) -> String { + let mut out = String::new(); + let mut rest = template; + for capture in captures { + let Some((head, tail)) = rest.split_once("{}") else { + break; + }; + out.push_str(head); + out.push_str(capture); + rest = tail; + } + out.push_str(rest); + out + } + + #[test] + fn template_reformat_is_lossless_and_smaller() { + let input = repetitive_template_log(); + let out = compress_templates(&input).expect("template reformat"); + + assert!(!out.lossy); + assert!(out.text.contains("[TOKENJUICE LOG TEMPLATE run=80")); + assert!(out.text.len() < input.len(), "{}", out.text); + assert_eq!( + reconstruct_template_reformat(&out.text).trim_end(), + input.trim_end() + ); + } + + #[tokio::test] + async fn template_reformat_routes_without_ccr() { + let input = repetitive_template_log(); + let hint = ContentHint { + explicit: Some(ContentKind::Log), + ..Default::default() + }; + let opts = CompressOptions { + min_bytes_to_compress: 64, + ccr_min_tokens: usize::MAX, + ..Default::default() + }; + + let out = crate::compress_content(&input, Some(hint), &opts).await; + + assert!(out.applied); + assert!(!out.lossy); + assert!(out.ccr_token.is_none()); + assert!(out.text.contains("[TOKENJUICE LOG TEMPLATE")); + } + + #[test] + fn template_reformat_declines_without_dynamic_fields() { + let mut input = String::new(); + for _ in 0..40 { + let _ = writeln!(input, "plain repeated line without dynamic fields"); + } + + assert!(compress_templates(&input).is_none()); + } + #[test] fn signal_keeps_errors_and_summary_drops_noise() { let input = noisy_log(); From 3e17f2a5e456b151b9b6d4fd47880bd2295b95b7 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Sun, 5 Jul 2026 09:11:52 +0000 Subject: [PATCH 007/120] Add TextCrusher and BM25 relevance --- src/compress.rs | 40 +++++ src/compressors/mod.rs | 12 +- src/compressors/text.rs | 370 ++++++++++++++++++++++++++++++++++++++++ src/lib.rs | 2 + src/relevance/bm25.rs | 167 ++++++++++++++++++ src/relevance/mod.rs | 5 + src/types.rs | 3 + 7 files changed, 592 insertions(+), 7 deletions(-) create mode 100644 src/compressors/text.rs create mode 100644 src/relevance/bm25.rs create mode 100644 src/relevance/mod.rs diff --git a/src/compress.rs b/src/compress.rs index e3678d0..fcd7779 100644 --- a/src/compress.rs +++ b/src/compress.rs @@ -390,6 +390,46 @@ mod tests { assert_eq!(store.get(token).as_deref(), Some(original.as_str())); } + #[tokio::test] + async fn textcrusher_output_offloads_original_for_recovery() { + let mut original = String::new(); + for i in 0..80 { + original.push_str(&format!( + "routine service note {i} describes ordinary progress through the worker queue.\n\n" + )); + } + original.push_str( + "ERROR sync.worker.v2 failed for REQUEST_ID 9F42 after retry 17 in api.worker.example.\n\n", + ); + for i in 80..160 { + original.push_str(&format!( + "routine service note {i} describes ordinary progress through the worker queue.\n\n" + )); + } + let hint = ContentHint { + explicit: Some(ContentKind::PlainText), + query: Some("sync.worker.v2 REQUEST_ID".to_string()), + ..Default::default() + }; + let store = cache::MemoryCcrStore::new(10, 1_000_000); + let mut opts = opts(); + opts.ccr_min_tokens = 1; + + let res = compress_content_with_store(&original, Some(hint), &opts, &store).await; + + assert!(res.applied); + assert!(res.lossy); + assert_eq!(res.content_kind, ContentKind::PlainText); + assert_eq!(res.compressor, CompressorKind::TextCrusher); + assert!( + res.body.contains("ERROR sync.worker.v2 failed"), + "{}", + res.body + ); + let token = res.ccr_token.as_deref().expect("offloaded"); + assert_eq!(store.get(token).as_deref(), Some(original.as_str())); + } + #[tokio::test] async fn exposes_body_and_recovery_footer_separately() { let mut rows = Vec::new(); diff --git a/src/compressors/mod.rs b/src/compressors/mod.rs index f5b0b74..a796fa0 100644 --- a/src/compressors/mod.rs +++ b/src/compressors/mod.rs @@ -17,6 +17,7 @@ pub mod log; pub mod ml_text; pub mod search; pub mod signals; +pub mod text; pub mod web_extract; use async_trait::async_trait; @@ -47,16 +48,13 @@ static LOG_COMPRESSOR: log::LogCompressor = log::LogCompressor; static SEARCH_COMPRESSOR: search::SearchCompressor = search::SearchCompressor; static DIFF_COMPRESSOR: diff::DiffCompressor = diff::DiffCompressor; static HTML_COMPRESSOR: html::HtmlCompressor = html::HtmlCompressor; -static ML_TEXT_COMPRESSOR: ml_text::MlTextCompressor = ml_text::MlTextCompressor; +static TEXT_COMPRESSOR: text::TextCompressor = text::TextCompressor; static GENERIC_COMPRESSOR: generic::GenericCompressor = generic::GenericCompressor; /// Map a detected [`ContentKind`] to the compressor that handles it. /// -/// `PlainText` routes to the ML compressor; whether it actually runs is gated -/// by `opts.ml_text_enabled` (and runtime Python/runtime_python_server -/// availability), and it falls back to [`generic::GenericCompressor`] otherwise -/// — that gating lives in [`crate::compress`], so this -/// function is a pure static mapping. +/// `PlainText` routes to the text compressor, which tries the optional ML path +/// first and then the deterministic extractive TextCrusher fallback. pub fn compressor_for(kind: ContentKind) -> &'static dyn Compressor { match kind { ContentKind::Json => &JSON_COMPRESSOR, @@ -65,7 +63,7 @@ pub fn compressor_for(kind: ContentKind) -> &'static dyn Compressor { ContentKind::Search => &SEARCH_COMPRESSOR, ContentKind::Diff => &DIFF_COMPRESSOR, ContentKind::Html => &HTML_COMPRESSOR, - ContentKind::PlainText => &ML_TEXT_COMPRESSOR, + ContentKind::PlainText => &TEXT_COMPRESSOR, } } diff --git a/src/compressors/text.rs b/src/compressors/text.rs new file mode 100644 index 0000000..f599739 --- /dev/null +++ b/src/compressors/text.rs @@ -0,0 +1,370 @@ +//! Plain-text compressor. +//! +//! ML compression is tried first when enabled. When it is unavailable or +//! disabled, TextCrusher provides a deterministic extractive fallback: it keeps +//! verbatim salient/query-relevant spans and suppresses near-duplicates. + +use async_trait::async_trait; +use std::collections::HashSet; + +use super::Compressor; +use crate::relevance::Bm25Corpus; +use crate::types::{CompressInput, CompressOptions, CompressOutput, CompressorKind}; + +pub const MIN_SEGMENTS: usize = 12; +pub const TARGET_RATIO: f32 = 0.40; +const MIN_TARGET_CHARS: usize = 800; + +pub struct TextCompressor; + +#[async_trait] +impl Compressor for TextCompressor { + fn kind(&self) -> CompressorKind { + CompressorKind::TextCrusher + } + + async fn compress( + &self, + input: &CompressInput<'_>, + opts: &CompressOptions, + ) -> Option { + if opts.ml_text_enabled { + match crate::ml::compress(input.content, opts).await { + Ok(Some(text)) if text.len() < input.content.len() => { + return Some(CompressOutput::lossy(text, CompressorKind::MlText)); + } + Ok(_) => {} + Err(e) => { + log::debug!("[tokenjuice][ml] unavailable, falling back: {e:#}"); + } + } + } + + compress_textcrusher(input.content, input.hint.query.as_deref(), opts) + } +} + +#[derive(Debug, Clone)] +struct Segment { + index: usize, + text: String, + terms: Vec, + score: f32, + salience: f32, +} + +pub fn compress_textcrusher( + content: &str, + query: Option<&str>, + opts: &CompressOptions, +) -> Option { + let mut segments = split_segments(content); + if segments.len() < MIN_SEGMENTS { + return None; + } + + let duplicate_pressure = duplicate_pressure(&segments); + let query = query.filter(|q| !q.trim().is_empty()); + let bm25 = query.map(|_| { + Bm25Corpus::from_tokenized( + segments + .iter() + .map(|segment| segment.terms.clone()) + .collect(), + ) + }); + + let segment_count = segments.len(); + for segment in &mut segments { + segment.salience = salience_score(&segment.text); + let recency = (segment.index + 1) as f32 / segment_count as f32; + let query_score = match (query, bm25.as_ref()) { + (Some(q), Some(corpus)) => corpus.score(q, segment.index).min(4.0) / 4.0, + _ => 0.0, + }; + segment.score = segment.salience * 2.0 + query_score * 2.5 + recency * 0.35; + } + + let has_signal = query.is_some() + || duplicate_pressure >= 4 + || segments.iter().any(|segment| segment.salience >= 0.5); + if !has_signal { + return None; + } + + let ratio_target = (content.len() as f32 * TARGET_RATIO) as usize; + let target_chars = opts + .max_inline_chars + .unwrap_or(ratio_target) + .min(ratio_target.max(MIN_TARGET_CHARS)) + .min(content.len().saturating_sub(1)); + if target_chars == 0 { + return None; + } + + let mut ranked: Vec = (0..segments.len()).collect(); + ranked.sort_by(|&a, &b| { + segments[b] + .score + .partial_cmp(&segments[a].score) + .unwrap_or(std::cmp::Ordering::Equal) + .then_with(|| a.cmp(&b)) + }); + + let mut selected = Vec::new(); + let mut selected_terms: Vec> = Vec::new(); + let mut used_chars = 0usize; + + for index in ranked { + let segment = &segments[index]; + if used_chars + segment.text.len() + 1 > target_chars && !selected.is_empty() { + continue; + } + let terms: HashSet = segment.terms.iter().cloned().collect(); + if selected_terms + .iter() + .any(|existing| jaccard(existing, &terms) >= 0.82) + { + continue; + } + used_chars += segment.text.len() + 1; + selected.push(index); + selected_terms.push(terms); + } + + if selected.is_empty() || selected.len() == segments.len() { + return None; + } + + selected.sort_unstable(); + let out = selected + .into_iter() + .map(|index| segments[index].text.as_str()) + .collect::>() + .join("\n"); + + if out.len() >= content.len() { + return None; + } + Some(CompressOutput::lossy(out, CompressorKind::TextCrusher)) +} + +fn split_segments(content: &str) -> Vec { + let mut segments = Vec::new(); + for block in content.split("\n\n") { + let trimmed = block.trim(); + if trimmed.is_empty() { + continue; + } + if trimmed.len() <= 260 { + push_segment(&mut segments, trimmed); + continue; + } + for sentence in split_sentences(trimmed) { + push_segment(&mut segments, sentence); + } + } + segments +} + +fn split_sentences(block: &str) -> Vec<&str> { + let mut out = Vec::new(); + let mut start = 0usize; + let mut iter = block.char_indices().peekable(); + while let Some((idx, ch)) = iter.next() { + let sentence_end = matches!(ch, '.' | '!' | '?') + && iter.peek().is_none_or(|(_, next)| next.is_whitespace()); + if !sentence_end { + continue; + } + let end = idx + ch.len_utf8(); + let sentence = block[start..end].trim(); + if !sentence.is_empty() { + out.push(sentence); + } + while let Some((next_idx, next_ch)) = iter.peek().copied() { + if next_ch.is_whitespace() { + iter.next(); + start = next_idx + next_ch.len_utf8(); + } else { + break; + } + } + } + let tail = block[start..].trim(); + if !tail.is_empty() { + out.push(tail); + } + out +} + +fn push_segment(segments: &mut Vec, text: &str) { + let terms = crate::relevance::tokenize(text); + if terms.len() < 3 { + return; + } + segments.push(Segment { + index: segments.len(), + text: text.to_string(), + terms, + score: 0.0, + salience: 0.0, + }); +} + +fn salience_score(text: &str) -> f32 { + let lower = text.to_ascii_lowercase(); + let mut score = 0.0f32; + if lower.contains("error") + || lower.contains("failed") + || lower.contains("panic") + || lower.contains("exception") + || lower.contains("warning") + || lower.contains("todo") + { + score += 1.0; + } + if text.chars().any(|ch| ch.is_ascii_digit()) { + score += 0.35; + } + if text.split_whitespace().any(is_all_caps_identifier) { + score += 0.5; + } + if text.split_whitespace().any(is_dotted_identifier) { + score += 0.5; + } + score.min(2.0) +} + +fn is_all_caps_identifier(token: &str) -> bool { + let token = token.trim_matches(|ch: char| !ch.is_ascii_alphanumeric() && ch != '_'); + token.len() >= 4 + && token.chars().any(|ch| ch.is_ascii_alphabetic()) + && token + .chars() + .all(|ch| ch.is_ascii_uppercase() || ch.is_ascii_digit() || ch == '_') +} + +fn is_dotted_identifier(token: &str) -> bool { + let token = token.trim_matches(|ch: char| !ch.is_ascii_alphanumeric() && ch != '.'); + token.split('.').filter(|part| part.len() >= 2).count() >= 2 +} + +fn duplicate_pressure(segments: &[Segment]) -> usize { + let mut seen: Vec> = Vec::new(); + let mut duplicates = 0usize; + for segment in segments { + let terms: HashSet = segment.terms.iter().cloned().collect(); + if seen + .iter() + .any(|existing| jaccard(existing, &terms) >= 0.82) + { + duplicates += 1; + } else { + seen.push(terms); + } + } + duplicates +} + +fn jaccard(a: &HashSet, b: &HashSet) -> f32 { + if a.is_empty() || b.is_empty() { + return 0.0; + } + let intersection = a.intersection(b).count(); + let union = a.len() + b.len() - intersection; + intersection as f32 / union as f32 +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::types::CompressOptions; + + fn opts() -> CompressOptions { + CompressOptions { + min_bytes_to_compress: 64, + ccr_min_tokens: 1, + ..Default::default() + } + } + + #[test] + fn keeps_errors_and_identifiers_as_verbatim_spans() { + let mut input = String::new(); + for i in 0..40 { + input.push_str(&format!( + "ordinary deployment progress line {i} with routine status information.\n\n" + )); + } + input.push_str("ERROR sync.worker.v2 failed for REQUEST_ID 9F42 after retry 17.\n\n"); + for i in 40..80 { + input.push_str(&format!( + "ordinary deployment progress line {i} with routine status information.\n\n" + )); + } + + let out = compress_textcrusher(&input, Some("sync.worker.v2 REQUEST_ID"), &opts()) + .expect("compresses") + .text; + + assert!(out.contains("ERROR sync.worker.v2 failed"), "{out}"); + for line in out.lines() { + assert!(input.contains(line), "non-verbatim span: {line}"); + } + } + + #[test] + fn suppresses_near_duplicate_prose_deterministically() { + let mut input = String::new(); + for i in 0..80 { + input.push_str(&format!( + "worker queue processed duplicate status message for tenant alpha batch {i}.\n\n" + )); + } + + let first = compress_textcrusher(&input, None, &opts()) + .expect("compresses") + .text; + let second = compress_textcrusher(&input, None, &opts()) + .expect("compresses") + .text; + + assert_eq!(first, second); + assert!(first.lines().count() < 20, "{first}"); + assert!(first.len() < input.len()); + } + + #[test] + fn plain_unsalient_data_declines() { + let sentences = [ + "soft morning light crossed the empty kitchen and settled on the table.", + "a folded blanket rested beside the chair near the quiet window.", + "the hallway carried a faint smell of cedar from the old cabinet.", + "afternoon clouds gathered slowly above the roofs beyond the lane.", + "a notebook remained open to a page of careful handwriting.", + "the garden path curved around stones set deep in the soil.", + "warm tea cooled beside a stack of letters tied with string.", + "the room held a simple lamp and a shelf of travel books.", + "a blue curtain moved lightly whenever the door opened.", + "distant bells marked the hour across the sleeping square.", + "the wooden floor showed pale marks from years of use.", + "fresh linen hung across the rail in the narrow courtyard.", + "a small map lay flat beneath a smooth glass paperweight.", + "evening settled over the street with a gentle amber color.", + "the desk drawer contained stamps envelopes and a fountain pen.", + "quiet footsteps faded on the stair beyond the landing.", + "the wall clock ticked steadily through the still apartment.", + "a weathered basket sat near the hearth filled with kindling.", + "the porch boards were dry after several days of sun.", + "a clean bowl and spoon waited beside the folded napkin.", + ]; + let mut input = String::new(); + for sentence in sentences { + input.push_str(sentence); + input.push_str("\n\n"); + } + + assert!(compress_textcrusher(&input, None, &opts()).is_none()); + } +} diff --git a/src/lib.rs b/src/lib.rs index d72fc92..5c5821b 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -18,6 +18,7 @@ pub mod openhuman; pub mod pipeline; pub mod policy; pub mod reduce; +pub mod relevance; pub mod rules; pub mod savings; pub mod text; @@ -56,6 +57,7 @@ pub use pipeline::{ }; pub use policy::{ShellCompactionPolicy, ShellPolicyDecision, apply_shell_compaction_policy}; pub use reduce::reduce_execution_with_rules; +pub use relevance::{Bm25Corpus, Bm25DocumentScore}; pub use rules::{LoadRuleOptions, load_builtin_rules, load_rules}; pub use tool_integration::{ CompactionStats, compact_output, compact_output_with_policy, compact_tool_output_with_policy, diff --git a/src/relevance/bm25.rs b/src/relevance/bm25.rs new file mode 100644 index 0000000..4f07869 --- /dev/null +++ b/src/relevance/bm25.rs @@ -0,0 +1,167 @@ +//! Dependency-free BM25 scorer. + +use std::collections::{HashMap, HashSet}; + +const K1: f32 = 1.2; +const B: f32 = 0.75; + +#[derive(Debug, Clone, PartialEq)] +pub struct Bm25DocumentScore { + pub index: usize, + pub score: f32, +} + +#[derive(Debug, Clone)] +pub struct Bm25Corpus { + docs: Vec>, + frequencies: Vec>, + document_frequency: HashMap, + average_len: f32, +} + +impl Bm25Corpus { + pub fn new<'a>(documents: impl IntoIterator) -> Self { + let docs: Vec> = documents.into_iter().map(tokenize).collect(); + Self::from_tokenized(docs) + } + + pub fn from_tokenized(docs: Vec>) -> Self { + let mut frequencies = Vec::with_capacity(docs.len()); + let mut document_frequency: HashMap = HashMap::new(); + let mut total_len = 0usize; + + for doc in &docs { + total_len += doc.len(); + let mut counts: HashMap = HashMap::new(); + for token in doc { + *counts.entry(token.clone()).or_insert(0) += 1; + } + let unique: HashSet<&String> = doc.iter().collect(); + for token in unique { + *document_frequency.entry(token.clone()).or_insert(0) += 1; + } + frequencies.push(counts); + } + + let average_len = if docs.is_empty() { + 0.0 + } else { + total_len as f32 / docs.len() as f32 + }; + + Self { + docs, + frequencies, + document_frequency, + average_len, + } + } + + pub fn score(&self, query: &str, index: usize) -> f32 { + let Some(doc) = self.docs.get(index) else { + return 0.0; + }; + if doc.is_empty() || self.average_len <= f32::EPSILON { + return 0.0; + } + + let query_tokens = tokenize(query); + if query_tokens.is_empty() { + return 0.0; + } + + let mut score = 0.0f32; + let doc_len = doc.len() as f32; + let doc_count = self.docs.len() as f32; + let counts = &self.frequencies[index]; + let unique_query: HashSet = query_tokens.into_iter().collect(); + + for token in unique_query { + let Some(&tf_count) = counts.get(&token) else { + continue; + }; + let df = *self.document_frequency.get(&token).unwrap_or(&0) as f32; + let idf = ((doc_count - df + 0.5) / (df + 0.5) + 1.0).ln(); + let tf = tf_count as f32; + let denom = tf + K1 * (1.0 - B + B * (doc_len / self.average_len)); + score += idf * (tf * (K1 + 1.0)) / denom; + } + + score + exact_identifier_boost(query, doc) + } + + pub fn score_all(&self, query: &str) -> Vec { + (0..self.docs.len()) + .map(|index| Bm25DocumentScore { + index, + score: self.score(query, index), + }) + .collect() + } +} + +pub fn tokenize(text: &str) -> Vec { + let mut tokens = Vec::new(); + let mut current = String::new(); + + for ch in text.chars() { + if ch.is_ascii_alphanumeric() || ch == '_' || ch == '-' || ch == '.' || ch == '@' { + current.push(ch.to_ascii_lowercase()); + } else { + push_token(&mut tokens, &mut current); + } + } + push_token(&mut tokens, &mut current); + tokens +} + +fn push_token(tokens: &mut Vec, current: &mut String) { + let token = current.trim_matches(|c: char| c == '-' || c == '.' || c == '@'); + if token.len() >= 2 { + tokens.push(token.to_string()); + } + current.clear(); +} + +fn exact_identifier_boost(query: &str, doc: &[String]) -> f32 { + tokenize(query) + .into_iter() + .filter(|token| is_identifier_like(token) && doc.iter().any(|doc_token| doc_token == token)) + .count() as f32 + * 1.5 +} + +fn is_identifier_like(token: &str) -> bool { + token.contains('_') + || token.contains('-') + || token.contains('.') + || token.contains('@') + || token.chars().any(|c| c.is_ascii_digit()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn exact_identifier_match_scores_above_generic_terms() { + let corpus = Bm25Corpus::new([ + "general retry handling around worker state", + "panic in sync_worker_v2 for account id 42", + "worker completed normally after queue drain", + ]); + + let scores = corpus.score_all("sync_worker_v2"); + + assert!(scores[1].score > scores[0].score, "{scores:?}"); + assert!(scores[1].score > scores[2].score, "{scores:?}"); + } + + #[test] + fn tokenization_keeps_dotted_and_email_identifiers() { + let tokens = tokenize("api.worker.example failed for ops@example.com"); + + assert!(tokens.contains(&"api.worker.example".to_string())); + assert!(tokens.contains(&"ops@example.com".to_string())); + } +} diff --git a/src/relevance/mod.rs b/src/relevance/mod.rs new file mode 100644 index 0000000..743bade --- /dev/null +++ b/src/relevance/mod.rs @@ -0,0 +1,5 @@ +//! Shared relevance scoring helpers for compressors and host adapters. + +pub mod bm25; + +pub use bm25::{Bm25Corpus, Bm25DocumentScore, tokenize}; diff --git a/src/types.rs b/src/types.rs index 1782f76..54bda5a 100644 --- a/src/types.rs +++ b/src/types.rs @@ -611,6 +611,8 @@ pub enum CompressorKind { Html, /// ML (Python/ModernBERT) plain-text compressor. MlText, + /// Deterministic extractive plain-text compressor. + TextCrusher, /// Line-oriented head/tail fallback. Generic, /// No compressor fired — pass-through. @@ -628,6 +630,7 @@ impl CompressorKind { CompressorKind::Diff => "diff", CompressorKind::Html => "html", CompressorKind::MlText => "ml_text", + CompressorKind::TextCrusher => "textcrusher", CompressorKind::Generic => "generic", CompressorKind::None => "none", } From fd2036e8ac18577f5d04287edb80769500a76a8f Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Sun, 5 Jul 2026 09:27:09 +0000 Subject: [PATCH 008/120] Upgrade SmartCrusher JSON planning --- src/compressors/json.rs | 421 ++++++++++++++++++++++++++++++++++------ 1 file changed, 364 insertions(+), 57 deletions(-) diff --git a/src/compressors/json.rs b/src/compressors/json.rs index af11e1d..6f2a8c0 100644 --- a/src/compressors/json.rs +++ b/src/compressors/json.rs @@ -14,12 +14,13 @@ //! so the dropped rows stay recoverable. use async_trait::async_trait; -use serde_json::Value; -use std::collections::BTreeSet; +use serde_json::{Map, Value}; +use std::collections::{BTreeMap, BTreeSet, HashMap}; use std::fmt::Write as _; use super::Compressor; use super::signals::has_error_indicators; +use crate::relevance::Bm25Corpus; use crate::types::{CompressInput, CompressOptions, CompressOutput, CompressorKind}; /// Minimum rows before tabular rendering is worth the header overhead. @@ -46,13 +47,51 @@ impl Compressor for JsonCompressor { input: &CompressInput<'_>, _opts: &CompressOptions, ) -> Option { - compress(input.content) + compress_with_query(input.content, input.hint.query.as_deref()) } } +#[derive(Debug, Clone, PartialEq)] +pub struct JsonAnalysis { + pub row_count: usize, + pub columns: Vec, + pub fields: Vec, + pub estimated_table_bytes: usize, +} + +#[derive(Debug, Clone, PartialEq)] +pub struct JsonFieldAnalysis { + pub key: String, + pub present: usize, + pub types: BTreeSet<&'static str>, + pub unique_count: usize, + pub constant: bool, + pub sparse: bool, + pub numeric: Option, +} + +#[derive(Debug, Clone, Copy, PartialEq)] +pub struct JsonNumericStats { + pub min: f64, + pub max: f64, + pub mean: f64, + pub stddev: f64, +} + +#[derive(Debug, Clone)] +struct JsonPlan { + lossy: bool, + keep_rows: Vec, +} + /// Compress a JSON array-of-objects into a compact table. Returns `None` when /// the content isn't a uniform-enough array of objects or wouldn't shrink. pub fn compress(content: &str) -> Option { + compress_with_query(content, None) +} + +/// Compress a JSON array-of-objects with optional query context for row anchors. +pub fn compress_with_query(content: &str, query: Option<&str>) -> Option { let value: Value = serde_json::from_str(content.trim()).ok()?; let array = value.as_array()?; if array.len() < MIN_ROWS { @@ -62,28 +101,19 @@ pub fn compress(content: &str) -> Option { return None; } - // Column order = first-seen key order across all rows (union, stable). - let mut columns: Vec = Vec::new(); - for item in array { - if let Some(obj) = item.as_object() { - for key in obj.keys() { - if !columns.iter().any(|c| c == key) { - columns.push(key.clone()); - } - } - } - } + let flat_rows = flatten_rows(array)?; + let analysis = analyze_flat_rows(&flat_rows); + let columns = analysis.columns.clone(); if columns.len() < 2 { return None; } // Render every row's cells up front so we can choose full vs. row-dropped. - let mut rows: Vec = Vec::with_capacity(array.len()); - for item in array { - let obj = item.as_object()?; + let mut rows: Vec = Vec::with_capacity(flat_rows.len()); + for obj in &flat_rows { let cells: Vec = columns .iter() - .map(|col| match obj.get(col) { + .map(|col| match obj.values.get(col) { None => String::new(), Some(v) => render_cell(v), }) @@ -91,7 +121,7 @@ pub fn compress(content: &str) -> Option { rows.push(cells.join(" | ")); } - let lossy = rows.len() > ROW_DROP_THRESHOLD; + let plan = plan_rows(&flat_rows, &analysis, query); let mut out = String::with_capacity(content.len()); let _ = writeln!( out, @@ -101,12 +131,11 @@ pub fn compress(content: &str) -> Option { ); let _ = writeln!(out, "{}", columns.join(" | ")); - if lossy { + if plan.lossy { // Keep head + tail PLUS any anomalous rows (errors / numeric outliers) // so the signal in a large homogeneous array survives row-dropping. - let keep = rows_to_keep(array, &columns, rows.len()); let mut prev: Option = None; - for &i in &keep { + for &i in &plan.keep_rows { if let Some(p) = prev { let gap = i - p - 1; if gap > 0 { @@ -138,11 +167,11 @@ pub fn compress(content: &str) -> Option { "[tokenjuice][json] {} rows × {} cols, lossy={} ({} -> {} bytes)", rows.len(), columns.len(), - lossy, + plan.lossy, content.len(), out.len(), ); - if lossy { + if plan.lossy { Some(CompressOutput::lossy(out, CompressorKind::SmartCrusher)) } else { // All values preserved, but the array→table reformat changes layout. @@ -153,62 +182,242 @@ pub fn compress(content: &str) -> Option { } } +/// Analyze a JSON array-of-objects without rendering it. +pub fn analyze(content: &str) -> Option { + let value: Value = serde_json::from_str(content.trim()).ok()?; + let array = value.as_array()?; + if array.len() < MIN_ROWS || !array.iter().all(Value::is_object) { + return None; + } + let flat_rows = flatten_rows(array)?; + Some(analyze_flat_rows(&flat_rows)) +} + /// Pick the row indices to keep when row-dropping: the head/tail windows plus /// any row flagged as anomalous (error text or a numeric outlier in any /// column). Returns ascending, de-duplicated indices. -fn rows_to_keep(array: &[Value], columns: &[String], n: usize) -> Vec { +fn plan_rows(rows: &[FlatRow], analysis: &JsonAnalysis, query: Option<&str>) -> JsonPlan { + let n = rows.len(); + let lossy = n > ROW_DROP_THRESHOLD; + if !lossy { + return JsonPlan { + lossy: false, + keep_rows: (0..n).collect(), + }; + } + let mut keep: BTreeSet = BTreeSet::new(); - for i in 0..HEAD_ROWS.min(n) { + for i in 0..dynamic_head_rows(n) { keep.insert(i); } - for i in n.saturating_sub(TAIL_ROWS)..n { + for i in n.saturating_sub(dynamic_tail_rows(n))..n { keep.insert(i); } // Error-text rows: any string/scalar cell carrying an error indicator. - for (i, item) in array.iter().enumerate() { - if let Some(obj) = item.as_object() { - let row_has_error = obj.values().any(|v| match v { - Value::String(s) => has_error_indicators(s), - other => has_error_indicators(&other.to_string()), - }); - if row_has_error { - keep.insert(i); - } + for (i, row) in rows.iter().enumerate() { + let row_has_error = row.values.values().any(|v| match v { + Value::String(s) => has_error_indicators(s), + other => has_error_indicators(&other.to_string()), + }); + if row_has_error { + keep.insert(i); } } // Numeric outliers: for each column, compute mean/std over numeric cells and // keep rows whose value is beyond OUTLIER_SIGMA. Bounded by a cap so a wide // anomalous tail can't defeat the point of dropping. - for col in columns { - let nums: Vec<(usize, f64)> = array - .iter() - .enumerate() - .filter_map(|(i, item)| { - item.as_object() - .and_then(|o| o.get(col)) - .and_then(Value::as_f64) - .map(|x| (i, x)) + for field in &analysis.fields { + if let Some(stats) = field.numeric + && stats.stddev > f64::EPSILON + { + for (i, row) in rows.iter().enumerate() { + if let Some(x) = row.values.get(&field.key).and_then(Value::as_f64) + && ((x - stats.mean) / stats.stddev).abs() >= OUTLIER_SIGMA + { + keep.insert(i); + } + } + } + if field.sparse { + for (i, row) in rows.iter().enumerate() { + if row.values.contains_key(&field.key) { + keep.insert(i); + } + } + } + } + + if let Some(query) = query.filter(|q| !q.trim().is_empty()) { + let docs: Vec = rows.iter().map(row_text).collect(); + let corpus = Bm25Corpus::new(docs.iter().map(String::as_str)); + let mut scored = corpus.score_all(query); + scored.sort_by(|a, b| { + b.score + .partial_cmp(&a.score) + .unwrap_or(std::cmp::Ordering::Equal) + .then_with(|| a.index.cmp(&b.index)) + }); + for score in scored + .into_iter() + .take(20) + .filter(|score| score.score > 0.0) + { + keep.insert(score.index); + } + } + + JsonPlan { + lossy, + keep_rows: keep.into_iter().collect(), + } +} + +#[derive(Debug, Clone)] +struct FlatRow { + values: BTreeMap, +} + +fn flatten_rows(array: &[Value]) -> Option> { + array + .iter() + .map(|item| { + let obj = item.as_object()?; + Some(FlatRow { + values: flatten_object(obj), }) - .collect(); - if nums.len() < 4 { - continue; + }) + .collect() +} + +fn flatten_object(obj: &Map) -> BTreeMap { + let mut out = BTreeMap::new(); + for (key, value) in obj { + flatten_value(key, value, &mut out); + } + out +} + +fn flatten_value(prefix: &str, value: &Value, out: &mut BTreeMap) { + match value { + Value::Object(obj) if !obj.is_empty() => { + for (key, child) in obj { + flatten_value(&format!("{prefix}.{key}"), child, out); + } } - let mean = nums.iter().map(|(_, x)| x).sum::() / nums.len() as f64; - let var = nums.iter().map(|(_, x)| (x - mean).powi(2)).sum::() / nums.len() as f64; - let std = var.sqrt(); - if std <= f64::EPSILON { - continue; + _ => { + out.insert(prefix.to_string(), value.clone()); } - for (i, x) in nums { - if ((x - mean) / std).abs() >= OUTLIER_SIGMA { - keep.insert(i); + } +} + +fn analyze_flat_rows(rows: &[FlatRow]) -> JsonAnalysis { + let mut present: HashMap = HashMap::new(); + let mut types: HashMap> = HashMap::new(); + let mut uniques: HashMap> = HashMap::new(); + let mut numeric_values: HashMap> = HashMap::new(); + + for row in rows { + for (key, value) in &row.values { + *present.entry(key.clone()).or_insert(0) += 1; + types + .entry(key.clone()) + .or_default() + .insert(value_type(value)); + uniques + .entry(key.clone()) + .or_default() + .insert(render_cell(value)); + if let Some(x) = value.as_f64() { + numeric_values.entry(key.clone()).or_default().push(x); } } } - keep.into_iter().collect() + let mut fields: Vec = present + .into_iter() + .map(|(key, present)| { + let unique_count = uniques.get(&key).map_or(0, BTreeSet::len); + JsonFieldAnalysis { + key: key.clone(), + present, + types: types.remove(&key).unwrap_or_default(), + unique_count, + constant: unique_count == 1 && present == rows.len(), + sparse: present <= sparse_threshold(rows.len()), + numeric: numeric_values.get(&key).map(|values| numeric_stats(values)), + } + }) + .collect(); + + fields.sort_by(|a, b| b.present.cmp(&a.present).then_with(|| a.key.cmp(&b.key))); + let columns = fields + .iter() + .map(|field| field.key.clone()) + .collect::>(); + let estimated_table_bytes = rows + .iter() + .map(|row| { + row.values + .values() + .map(render_cell) + .map(|s| s.len() + 3) + .sum::() + }) + .sum::() + + columns.iter().map(String::len).sum::(); + + JsonAnalysis { + row_count: rows.len(), + columns, + fields, + estimated_table_bytes, + } +} + +fn value_type(value: &Value) -> &'static str { + match value { + Value::Null => "null", + Value::Bool(_) => "bool", + Value::Number(_) => "number", + Value::String(_) => "string", + Value::Array(_) => "array", + Value::Object(_) => "object", + } +} + +fn numeric_stats(values: &[f64]) -> JsonNumericStats { + let min = values.iter().copied().fold(f64::INFINITY, f64::min); + let max = values.iter().copied().fold(f64::NEG_INFINITY, f64::max); + let mean = values.iter().sum::() / values.len() as f64; + let variance = values.iter().map(|x| (x - mean).powi(2)).sum::() / values.len() as f64; + JsonNumericStats { + min, + max, + mean, + stddev: variance.sqrt(), + } +} + +fn sparse_threshold(row_count: usize) -> usize { + 3.max(row_count / 10) +} + +fn dynamic_head_rows(row_count: usize) -> usize { + HEAD_ROWS.min((row_count / 4).max(5)) +} + +fn dynamic_tail_rows(row_count: usize) -> usize { + TAIL_ROWS.min((row_count / 8).max(3)) +} + +fn row_text(row: &FlatRow) -> String { + row.values + .iter() + .map(|(key, value)| format!("{key}={}", render_cell(value))) + .collect::>() + .join(" ") } /// Render a single cell. Scalars print bare-ish; nested values stay as compact @@ -258,6 +467,104 @@ mod tests { assert!(c.text.len() < input.len()); } + #[test] + fn analyzer_reports_constants_sparse_fields_and_numeric_stats() { + let mut rows = Vec::new(); + for i in 0..20 { + let extra = if i == 13 { + r#","debug":"only here""# + } else { + "" + }; + rows.push(format!( + r#"{{"id":{i},"status":"active","latency":{}{extra}}}"#, + 10 + i + )); + } + let input = format!("[{}]", rows.join(",")); + + let analysis = analyze(&input).expect("analysis"); + let status = analysis + .fields + .iter() + .find(|field| field.key == "status") + .expect("status field"); + let debug = analysis + .fields + .iter() + .find(|field| field.key == "debug") + .expect("debug field"); + let latency = analysis + .fields + .iter() + .find(|field| field.key == "latency") + .expect("latency field"); + + assert!(status.constant); + assert!(debug.sparse); + assert_eq!(debug.present, 1); + assert_eq!(latency.numeric.expect("numeric").min, 10.0); + } + + #[test] + fn query_anchor_row_survives_dropped_middle() { + let mut rows = Vec::new(); + for i in 0..140 { + let note = if i == 71 { + "contains special needle token" + } else { + "ordinary row" + }; + rows.push(format!( + r#"{{"id":{i},"name":"record {i}","status":"active","note":"{note}"}}"# + )); + } + let input = format!("[{}]", rows.join(",")); + + let c = compress_with_query(&input, Some("special needle")).expect("compresses"); + + assert!(c.lossy); + assert!(c.text.contains("special needle token"), "{}", c.text); + } + + #[test] + fn sparse_structural_row_survives_dropped_middle() { + let mut rows = Vec::new(); + for i in 0..140 { + let extra = if i == 72 { + r#","diagnostic":"rare sparse field""# + } else { + "" + }; + rows.push(format!( + r#"{{"id":{i},"name":"record {i}","status":"active"{extra}}}"# + )); + } + let input = format!("[{}]", rows.join(",")); + + let c = compress(&input).expect("compresses"); + + assert!(c.lossy); + assert!(c.text.contains("rare sparse field"), "{}", c.text); + } + + #[test] + fn nested_objects_flatten_to_dotted_columns() { + let mut rows = Vec::new(); + for i in 0..20 { + rows.push(format!( + r#"{{"id":{i},"user":{{"name":"user {i}","team":"core"}},"status":"active"}}"# + )); + } + let input = format!("[{}]", rows.join(",")); + + let c = compress(&input).expect("compresses"); + + assert!(c.text.contains("user.name"), "{}", c.text); + assert!(c.text.contains("user.team"), "{}", c.text); + assert!(c.text.contains("user 7"), "{}", c.text); + } + #[test] fn keeps_error_row_in_dropped_middle() { // A homogeneous array with a single error row buried in the middle: the From 49b3ed9806684d4ac1ea425410e0a6bade9a5855 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Sun, 5 Jul 2026 09:40:05 +0000 Subject: [PATCH 009/120] Protect custom tags in ML text compression --- src/compressors/ml_text.rs | 15 +- src/compressors/text.rs | 471 ++++++++++++++++++++++++++++++++++++- 2 files changed, 463 insertions(+), 23 deletions(-) diff --git a/src/compressors/ml_text.rs b/src/compressors/ml_text.rs index 9f0666f..14bc59b 100644 --- a/src/compressors/ml_text.rs +++ b/src/compressors/ml_text.rs @@ -14,6 +14,7 @@ use async_trait::async_trait; use super::Compressor; +use crate::compressors::text::compress_ml_with_tag_protection; use crate::types::{CompressInput, CompressOptions, CompressOutput, CompressorKind}; pub struct MlTextCompressor; @@ -29,18 +30,6 @@ impl Compressor for MlTextCompressor { input: &CompressInput<'_>, opts: &CompressOptions, ) -> Option { - if !opts.ml_text_enabled { - return None; - } - match crate::ml::compress(input.content, opts).await { - Ok(Some(text)) if text.len() < input.content.len() => { - Some(CompressOutput::lossy(text, CompressorKind::MlText)) - } - Ok(_) => None, - Err(e) => { - log::debug!("[tokenjuice][ml] unavailable, falling back: {e:#}"); - None - } - } + compress_ml_with_tag_protection(input.content, opts).await } } diff --git a/src/compressors/text.rs b/src/compressors/text.rs index f599739..c3e517e 100644 --- a/src/compressors/text.rs +++ b/src/compressors/text.rs @@ -28,16 +28,8 @@ impl Compressor for TextCompressor { input: &CompressInput<'_>, opts: &CompressOptions, ) -> Option { - if opts.ml_text_enabled { - match crate::ml::compress(input.content, opts).await { - Ok(Some(text)) if text.len() < input.content.len() => { - return Some(CompressOutput::lossy(text, CompressorKind::MlText)); - } - Ok(_) => {} - Err(e) => { - log::debug!("[tokenjuice][ml] unavailable, falling back: {e:#}"); - } - } + if let Some(out) = compress_ml_with_tag_protection(input.content, opts).await { + return Some(out); } compress_textcrusher(input.content, input.hint.query.as_deref(), opts) @@ -53,6 +45,32 @@ struct Segment { salience: f32, } +pub async fn compress_ml_with_tag_protection( + content: &str, + opts: &CompressOptions, +) -> Option { + if !opts.ml_text_enabled { + return None; + } + + let protected = protect_custom_tags(content); + match crate::ml::compress(&protected.text, opts).await { + Ok(Some(text)) => { + let text = protected.restore(&text)?; + if text.len() < content.len() { + Some(CompressOutput::lossy(text, CompressorKind::MlText)) + } else { + None + } + } + Ok(_) => None, + Err(e) => { + log::debug!("[tokenjuice][ml] unavailable, falling back: {e:#}"); + None + } + } +} + pub fn compress_textcrusher( content: &str, query: Option<&str>, @@ -149,6 +167,327 @@ pub fn compress_textcrusher( Some(CompressOutput::lossy(out, CompressorKind::TextCrusher)) } +#[derive(Debug, Clone, PartialEq, Eq)] +struct ProtectedText { + text: String, + replacements: Vec, +} + +impl ProtectedText { + fn restore(&self, candidate: &str) -> Option { + for replacement in &self.replacements { + if !candidate.contains(&replacement.placeholder) { + return None; + } + } + + let mut restored = candidate.to_string(); + for replacement in &self.replacements { + restored = restored.replace(&replacement.placeholder, &replacement.original); + } + Some(restored) + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +struct TagReplacement { + placeholder: String, + original: String, +} + +fn protect_custom_tags(content: &str) -> ProtectedText { + let ranges = custom_tag_ranges(content); + if ranges.is_empty() { + return ProtectedText { + text: content.to_string(), + replacements: Vec::new(), + }; + } + + let prefix = safe_placeholder_prefix(content); + let mut text = String::with_capacity(content.len()); + let mut replacements = Vec::new(); + let mut cursor = 0usize; + + for (i, range) in ranges.into_iter().enumerate() { + if range.start < cursor { + continue; + } + text.push_str(&content[cursor..range.start]); + let placeholder = format!("{prefix}{i}__"); + text.push_str(&placeholder); + replacements.push(TagReplacement { + placeholder, + original: content[range.start..range.end].to_string(), + }); + cursor = range.end; + } + text.push_str(&content[cursor..]); + + ProtectedText { text, replacements } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +struct ByteRange { + start: usize, + end: usize, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +struct OpenTag { + name: String, + start: usize, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum TagKind { + Open, + Close, + SelfClosing, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +struct ParsedTag { + range: ByteRange, + name: String, + kind: TagKind, + custom: bool, +} + +fn custom_tag_ranges(content: &str) -> Vec { + let mut ranges = Vec::new(); + let mut stack: Vec = Vec::new(); + let mut cursor = 0usize; + + while let Some(relative) = content[cursor..].find('<') { + let start = cursor + relative; + let Some(tag) = parse_tag_at(content, start) else { + cursor = start + 1; + continue; + }; + cursor = tag.range.end; + + if !tag.custom { + continue; + } + + match tag.kind { + TagKind::SelfClosing => ranges.push(tag.range), + TagKind::Open => stack.push(OpenTag { + name: tag.name, + start: tag.range.start, + }), + TagKind::Close => { + if let Some(pos) = stack.iter().rposition(|open| open.name == tag.name) { + let open = stack.remove(pos); + stack.truncate(pos); + ranges.push(ByteRange { + start: open.start, + end: tag.range.end, + }); + } + } + } + } + + ranges.sort_by(|a, b| a.start.cmp(&b.start).then_with(|| b.end.cmp(&a.end))); + let mut non_overlapping = Vec::new(); + let mut covered_end = 0usize; + for range in ranges { + if range.start >= covered_end { + covered_end = range.end; + non_overlapping.push(range); + } + } + non_overlapping +} + +fn parse_tag_at(content: &str, start: usize) -> Option { + let tail = content.get(start..)?; + if !tail.starts_with('<') + || tail.starts_with(""; +const TINYJUICE_MANAGED_END: &str = ""; + fn main() -> ExitCode { match run(std::env::args().skip(1).collect()) { Ok(()) => ExitCode::SUCCESS, @@ -37,6 +40,8 @@ fn run(args: Vec) -> Result<(), String> { "cat" => run_cat(&args[1..]), "stats" => run_stats(&args[1..]), "doctor" => run_doctor(&args[1..]), + "install" => run_install(&args[1..]), + "uninstall" => run_uninstall(&args[1..]), "-h" | "--help" | "help" => { print_usage(); Ok(()) @@ -115,8 +120,11 @@ fn run_stats(args: &[String]) -> Result<(), String> { } fn run_doctor(args: &[String]) -> Result<(), String> { + let mut target = None; let mut store_dir = None; let mut fixtures_dir = PathBuf::from("tests/fixtures"); + let mut codex_home = None; + let mut openhuman_root = None; let mut check_fixtures = false; let mut pretty = false; let mut i = 0; @@ -130,6 +138,20 @@ fn run_doctor(args: &[String]) -> Result<(), String> { .ok_or_else(|| "--store-dir requires a value".to_owned())?, )); } + "--codex-home" => { + i += 1; + codex_home = Some(PathBuf::from( + args.get(i) + .ok_or_else(|| "--codex-home requires a value".to_owned())?, + )); + } + "--openhuman-root" => { + i += 1; + openhuman_root = + Some(PathBuf::from(args.get(i).ok_or_else(|| { + "--openhuman-root requires a value".to_owned() + })?)); + } "--fixtures" => { check_fixtures = true; if let Some(next) = args.get(i + 1) @@ -145,11 +167,22 @@ fn run_doctor(args: &[String]) -> Result<(), String> { print_doctor_usage(); return Ok(()); } - value => return Err(format!("unknown doctor option: {value}")), + value if value.starts_with('-') => { + return Err(format!("unknown doctor option: {value}")); + } + value => { + if target.replace(value.to_owned()).is_some() { + return Err("doctor accepts at most one host target".to_owned()); + } + } } i += 1; } + if let Some(target) = target { + return run_host_doctor(&target, pretty, codex_home, openhuman_root); + } + let mut status = "disabled"; let mut checks = Vec::new(); @@ -264,6 +297,393 @@ fn run_doctor(args: &[String]) -> Result<(), String> { } } +fn run_host_doctor( + target: &str, + pretty: bool, + codex_home: Option, + openhuman_root: Option, +) -> Result<(), String> { + let checks = match target { + "codex" => vec![doctor_codex(codex_home)], + "openhuman" => vec![doctor_openhuman(openhuman_root)], + "hooks" => vec![doctor_codex(codex_home), doctor_openhuman(openhuman_root)], + other => return Err(format!("unknown doctor host target: {other}")), + }; + let status = checks + .iter() + .filter_map(|check| check.get("status").and_then(serde_json::Value::as_str)) + .fold("disabled", merge_doctor_status); + let report = serde_json::json!({ + "status": status, + "checks": checks, + }); + if pretty { + println!( + "{}", + serde_json::to_string_pretty(&report) + .map_err(|error| format!("failed to serialize doctor report: {error}"))? + ); + } else { + println!( + "{}", + serde_json::to_string(&report) + .map_err(|error| format!("failed to serialize doctor report: {error}"))? + ); + } + + if status == "broken" { + Err("doctor found broken host checks".to_owned()) + } else { + Ok(()) + } +} + +fn run_install(args: &[String]) -> Result<(), String> { + if args.iter().any(|arg| arg == "-h" || arg == "--help") { + print_install_usage(); + return Ok(()); + } + let install = parse_host_mutation_args("install", args, true)?; + match install.host.as_str() { + "codex" => { + let report = install_codex(install.codex_home, install.local_binary)?; + println!( + "{}", + serde_json::to_string(&report) + .map_err(|error| format!("failed to serialize install report: {error}"))? + ); + Ok(()) + } + other => Err(format!("unknown install host: {other}")), + } +} + +fn run_uninstall(args: &[String]) -> Result<(), String> { + if args.iter().any(|arg| arg == "-h" || arg == "--help") { + print_uninstall_usage(); + return Ok(()); + } + let uninstall = parse_host_mutation_args("uninstall", args, false)?; + match uninstall.host.as_str() { + "codex" => { + let report = uninstall_codex(uninstall.codex_home)?; + println!( + "{}", + serde_json::to_string(&report) + .map_err(|error| format!("failed to serialize uninstall report: {error}"))? + ); + Ok(()) + } + other => Err(format!("unknown uninstall host: {other}")), + } +} + +struct HostMutationArgs { + host: String, + codex_home: Option, + local_binary: Option, +} + +fn parse_host_mutation_args( + command: &str, + args: &[String], + allow_local: bool, +) -> Result { + let mut host = None; + let mut codex_home = None; + let mut local_binary = None; + let mut i = 0; + + while i < args.len() { + match args[i].as_str() { + "--codex-home" => { + i += 1; + codex_home = Some(PathBuf::from( + args.get(i) + .ok_or_else(|| "--codex-home requires a value".to_owned())?, + )); + } + "--local" if allow_local => { + i += 1; + local_binary = + Some(PathBuf::from(args.get(i).ok_or_else(|| { + "--local requires a binary path".to_owned() + })?)); + } + value if value.starts_with('-') => { + return Err(format!("unknown {command} option: {value}")); + } + value => { + if host.replace(value.to_owned()).is_some() { + return Err(format!("{command} accepts exactly one host")); + } + } + } + i += 1; + } + + Ok(HostMutationArgs { + host: host.ok_or_else(|| format!("{command} requires a host"))?, + codex_home, + local_binary, + }) +} + +fn install_codex( + codex_home: Option, + local_binary: Option, +) -> Result { + let root = codex_home.unwrap_or_else(default_codex_home); + let instruction_path = codex_instruction_path(&root); + let binary = local_binary + .map(|path| path.to_string_lossy().into_owned()) + .unwrap_or_else(|| "tinyjuice".to_owned()); + let block = managed_codex_block(&binary); + let original = match fs::read_to_string(&instruction_path) { + Ok(content) => content, + Err(error) if error.kind() == io::ErrorKind::NotFound => String::new(), + Err(error) => { + return Err(format!( + "failed to read {}: {error}", + instruction_path.display() + )); + } + }; + let updated = replace_or_insert_managed_block(&original, &block); + let changed = updated != original; + if changed { + write_managed_file(&instruction_path, &original, &updated)?; + } + Ok(serde_json::json!({ + "host": "codex", + "status": "ok", + "changed": changed, + "configPath": instruction_path.to_string_lossy(), + "command": format!("{binary} reduce-json"), + })) +} + +fn uninstall_codex(codex_home: Option) -> Result { + let root = codex_home.unwrap_or_else(default_codex_home); + let instruction_path = codex_instruction_path(&root); + let original = match fs::read_to_string(&instruction_path) { + Ok(content) => content, + Err(error) if error.kind() == io::ErrorKind::NotFound => { + return Ok(serde_json::json!({ + "host": "codex", + "status": "ok", + "changed": false, + "configPath": instruction_path.to_string_lossy(), + })); + } + Err(error) => { + return Err(format!( + "failed to read {}: {error}", + instruction_path.display() + )); + } + }; + let updated = remove_managed_block(&original); + let changed = updated != original; + if changed { + write_managed_file(&instruction_path, &original, &updated)?; + } + Ok(serde_json::json!({ + "host": "codex", + "status": "ok", + "changed": changed, + "configPath": instruction_path.to_string_lossy(), + })) +} + +fn doctor_codex(codex_home: Option) -> serde_json::Value { + let root = codex_home.unwrap_or_else(default_codex_home); + let instruction_path = codex_instruction_path(&root); + let repair_command = "tinyjuice install codex".to_owned(); + let expected_command = "tinyjuice reduce-json".to_owned(); + + let content = match fs::read_to_string(&instruction_path) { + Ok(content) => content, + Err(error) if error.kind() == io::ErrorKind::NotFound => { + return serde_json::json!({ + "name": "codex", + "status": "disabled", + "configPath": instruction_path.to_string_lossy(), + "expectedCommand": expected_command, + "repairCommand": repair_command, + }); + } + Err(error) => { + return serde_json::json!({ + "name": "codex", + "status": "broken", + "configPath": instruction_path.to_string_lossy(), + "expectedCommand": expected_command, + "repairCommand": repair_command, + "error": format!("failed to read Codex TinyJuice instructions: {error}"), + }); + } + }; + + let Some(block) = managed_block(&content) else { + return serde_json::json!({ + "name": "codex", + "status": "disabled", + "configPath": instruction_path.to_string_lossy(), + "expectedCommand": expected_command, + "repairCommand": repair_command, + }); + }; + let detected_command = managed_command(block); + let status = match detected_command.as_deref().and_then(command_binary_status) { + Some(false) => "broken", + _ => "ok", + }; + let mut check = serde_json::json!({ + "name": "codex", + "status": status, + "configPath": instruction_path.to_string_lossy(), + "expectedCommand": expected_command, + "detectedCommand": detected_command, + "repairCommand": repair_command, + }); + if status == "broken" { + check["error"] = + serde_json::Value::String("managed command points at a missing executable".to_owned()); + } + check +} + +fn doctor_openhuman(openhuman_root: Option) -> serde_json::Value { + let root = openhuman_root.unwrap_or_else(default_openhuman_root); + let adapter_path = root + .join("src") + .join("openhuman") + .join("tokenjuice") + .join("mod.rs"); + if adapter_path.is_file() { + serde_json::json!({ + "name": "openhuman", + "status": "ok", + "configPath": adapter_path.to_string_lossy(), + "expectedCommand": "direct Rust crate integration", + "detectedCommand": "src/openhuman/tokenjuice", + "repairCommand": "tinyjuice doctor openhuman --openhuman-root PATH", + }) + } else { + serde_json::json!({ + "name": "openhuman", + "status": "disabled", + "configPath": adapter_path.to_string_lossy(), + "expectedCommand": "direct Rust crate integration", + "repairCommand": "tinyjuice doctor openhuman --openhuman-root PATH", + }) + } +} + +fn default_codex_home() -> PathBuf { + std::env::var_os("CODEX_HOME") + .map(PathBuf::from) + .or_else(|| std::env::var_os("HOME").map(|home| PathBuf::from(home).join(".codex"))) + .unwrap_or_else(|| PathBuf::from(".codex")) +} + +fn default_openhuman_root() -> PathBuf { + PathBuf::from("../openhuman-4") +} + +fn codex_instruction_path(root: &Path) -> PathBuf { + root.join("instructions").join("tinyjuice.md") +} + +fn managed_codex_block(binary: &str) -> String { + format!( + "{TINYJUICE_MANAGED_BEGIN}\n\ + # TinyJuice\n\ + Command: {binary} reduce-json\n\ + When tool output is too large for context, prefer passing a structured payload to the command above or using `tinyjuice wrap -- ` when no post-tool hook is available.\n\ + {TINYJUICE_MANAGED_END}" + ) +} + +fn managed_block(content: &str) -> Option<&str> { + let start = content.find(TINYJUICE_MANAGED_BEGIN)? + TINYJUICE_MANAGED_BEGIN.len(); + let end = content[start..].find(TINYJUICE_MANAGED_END)? + start; + Some(content[start..end].trim()) +} + +fn replace_or_insert_managed_block(content: &str, block: &str) -> String { + if let Some((start, end)) = managed_block_bounds(content) { + let mut out = String::with_capacity(content.len() + block.len()); + out.push_str(&content[..start]); + out.push_str(block); + out.push_str(&content[end..]); + out + } else if content.trim().is_empty() { + format!("{block}\n") + } else { + format!("{}\n\n{block}\n", content.trim_end()) + } +} + +fn remove_managed_block(content: &str) -> String { + if let Some((start, end)) = managed_block_bounds(content) { + let mut out = String::with_capacity(content.len()); + out.push_str(content[..start].trim_end()); + out.push_str(content[end..].trim_start_matches('\n')); + if !out.is_empty() && !out.ends_with('\n') { + out.push('\n'); + } + out + } else { + content.to_owned() + } +} + +fn managed_block_bounds(content: &str) -> Option<(usize, usize)> { + let start = content.find(TINYJUICE_MANAGED_BEGIN)?; + let end = content[start..].find(TINYJUICE_MANAGED_END)? + start + TINYJUICE_MANAGED_END.len(); + Some((start, end)) +} + +fn write_managed_file(path: &Path, original: &str, updated: &str) -> Result<(), String> { + let parent = path + .parent() + .ok_or_else(|| format!("{} has no parent directory", path.display()))?; + fs::create_dir_all(parent) + .map_err(|error| format!("failed to create {}: {error}", parent.display()))?; + if path.exists() && !original.is_empty() { + let backup = path.with_extension("md.bak"); + if !backup.exists() { + fs::write(&backup, original) + .map_err(|error| format!("failed to write backup {}: {error}", backup.display()))?; + } + } + let tmp = path.with_extension("md.tmp"); + fs::write(&tmp, updated) + .map_err(|error| format!("failed to write temp file {}: {error}", tmp.display()))?; + fs::rename(&tmp, path).map_err(|error| format!("failed to replace {}: {error}", path.display())) +} + +fn managed_command(block: &str) -> Option { + block.lines().find_map(|line| { + line.trim() + .strip_prefix("Command:") + .map(str::trim) + .filter(|command| !command.is_empty()) + .map(str::to_owned) + }) +} + +fn command_binary_status(command: &str) -> Option { + let binary = command.split_whitespace().next()?; + if binary.contains(std::path::MAIN_SEPARATOR) || Path::new(binary).is_absolute() { + Some(Path::new(binary).is_file()) + } else { + None + } +} + fn run_cat(args: &[String]) -> Result<(), String> { let mut store_dir = None; let mut token = None; @@ -423,11 +843,12 @@ fn read_ccr_store_inventory(store_dir: &Path) -> Result &'static str { - if doctor_status_rank(next) > doctor_status_rank(current) { - next - } else { - current +fn merge_doctor_status(current: &str, next: &str) -> &'static str { + match doctor_status_rank(current).max(doctor_status_rank(next)) { + 3 => "broken", + 2 => "warn", + 1 => "ok", + _ => "disabled", } } @@ -820,6 +1241,8 @@ fn print_usage() { println!(" cat Print a token from an explicit CCR disk store"); println!(" stats Report metadata-only CCR disk store stats"); println!(" doctor Report TinyJuice health checks"); + println!(" install Install TinyJuice integration for a host"); + println!(" uninstall Remove TinyJuice integration from a host"); } fn print_reduce_usage() { @@ -860,5 +1283,15 @@ fn print_stats_usage() { } fn print_doctor_usage() { - println!("Usage: tinyjuice doctor [--pretty] [--fixtures [dir]] [--store-dir DIR]"); + println!( + "Usage: tinyjuice doctor [host|hooks] [--pretty] [--fixtures [dir]] [--store-dir DIR] [--codex-home DIR] [--openhuman-root DIR]" + ); +} + +fn print_install_usage() { + println!("Usage: tinyjuice install codex [--codex-home DIR] [--local PATH]"); +} + +fn print_uninstall_usage() { + println!("Usage: tinyjuice uninstall codex [--codex-home DIR]"); } diff --git a/tests/cli.rs b/tests/cli.rs index 187e1b0..9f859fa 100644 --- a/tests/cli.rs +++ b/tests/cli.rs @@ -372,3 +372,203 @@ fn doctor_reports_broken_store_dir() { assert_eq!(store["status"], "broken"); assert_eq!(store["storeDir"], missing_dir); } + +#[test] +fn doctor_codex_reports_disabled_without_managed_instructions() { + let codex_home = tempfile::tempdir().expect("temp codex home"); + + let output = tinyjuice() + .args([ + "doctor", + "codex", + "--codex-home", + &codex_home.path().to_string_lossy(), + ]) + .output() + .expect("run tinyjuice doctor codex"); + + assert!(output.status.success(), "{output:#?}"); + let response: serde_json::Value = serde_json::from_slice(&output.stdout).expect("doctor json"); + assert_eq!(response["status"], "disabled"); + assert_eq!(response["checks"][0]["name"], "codex"); + assert_eq!(response["checks"][0]["status"], "disabled"); + assert_eq!( + response["checks"][0]["repairCommand"], + "tinyjuice install codex" + ); +} + +#[test] +fn doctor_codex_reports_ok_for_existing_managed_command() { + let codex_home = tempfile::tempdir().expect("temp codex home"); + let instructions_dir = codex_home.path().join("instructions"); + fs::create_dir(&instructions_dir).expect("create instructions dir"); + let command = format!("{} reduce-json", env!("CARGO_BIN_EXE_tinyjuice")); + fs::write( + instructions_dir.join("tinyjuice.md"), + format!( + "user notes\n\nCommand: {command}\n\n" + ), + ) + .expect("write managed instructions"); + + let output = tinyjuice() + .args([ + "doctor", + "codex", + "--codex-home", + &codex_home.path().to_string_lossy(), + ]) + .output() + .expect("run tinyjuice doctor codex"); + + assert!(output.status.success(), "{output:#?}"); + let response: serde_json::Value = serde_json::from_slice(&output.stdout).expect("doctor json"); + assert_eq!(response["status"], "ok"); + assert_eq!(response["checks"][0]["status"], "ok"); + assert_eq!(response["checks"][0]["detectedCommand"], command); +} + +#[test] +fn doctor_codex_reports_broken_for_missing_managed_command() { + let codex_home = tempfile::tempdir().expect("temp codex home"); + let instructions_dir = codex_home.path().join("instructions"); + fs::create_dir(&instructions_dir).expect("create instructions dir"); + let command = codex_home + .path() + .join("missing-tinyjuice") + .to_string_lossy() + .into_owned(); + fs::write( + instructions_dir.join("tinyjuice.md"), + format!( + "\nCommand: {command} reduce-json\n\n" + ), + ) + .expect("write managed instructions"); + + let output = tinyjuice() + .args([ + "doctor", + "codex", + "--codex-home", + &codex_home.path().to_string_lossy(), + ]) + .output() + .expect("run tinyjuice doctor codex"); + + assert!(!output.status.success(), "{output:#?}"); + let response: serde_json::Value = serde_json::from_slice(&output.stdout).expect("doctor json"); + assert_eq!(response["status"], "broken"); + assert_eq!(response["checks"][0]["status"], "broken"); + assert_eq!( + response["checks"][0]["error"], + "managed command points at a missing executable" + ); +} + +#[test] +fn install_codex_writes_managed_block_and_is_idempotent() { + let codex_home = tempfile::tempdir().expect("temp codex home"); + let local = env!("CARGO_BIN_EXE_tinyjuice"); + + let first = tinyjuice() + .args([ + "install", + "codex", + "--codex-home", + &codex_home.path().to_string_lossy(), + "--local", + local, + ]) + .output() + .expect("run tinyjuice install codex"); + assert!(first.status.success(), "{first:#?}"); + let first_json: serde_json::Value = + serde_json::from_slice(&first.stdout).expect("install json"); + assert_eq!(first_json["status"], "ok"); + assert_eq!(first_json["changed"], true); + + let second = tinyjuice() + .args([ + "install", + "codex", + "--codex-home", + &codex_home.path().to_string_lossy(), + "--local", + local, + ]) + .output() + .expect("rerun tinyjuice install codex"); + assert!(second.status.success(), "{second:#?}"); + let second_json: serde_json::Value = + serde_json::from_slice(&second.stdout).expect("install json"); + assert_eq!(second_json["changed"], false); + + let instructions = + fs::read_to_string(codex_home.path().join("instructions/tinyjuice.md")).expect("read file"); + assert!(instructions.contains("")); + assert!(instructions.contains(&format!("Command: {local} reduce-json"))); +} + +#[test] +fn install_codex_preserves_unrelated_text_and_creates_backup() { + let codex_home = tempfile::tempdir().expect("temp codex home"); + let instructions_dir = codex_home.path().join("instructions"); + fs::create_dir(&instructions_dir).expect("create instructions dir"); + let path = instructions_dir.join("tinyjuice.md"); + fs::write(&path, "keep this note\n").expect("write existing instructions"); + + let output = tinyjuice() + .args([ + "install", + "codex", + "--codex-home", + &codex_home.path().to_string_lossy(), + "--local", + env!("CARGO_BIN_EXE_tinyjuice"), + ]) + .output() + .expect("run tinyjuice install codex"); + + assert!(output.status.success(), "{output:#?}"); + let updated = fs::read_to_string(&path).expect("read updated instructions"); + assert!(updated.contains("keep this note"), "{updated}"); + assert!(updated.contains("tinyjuice-managed:start"), "{updated}"); + assert_eq!( + fs::read_to_string(path.with_extension("md.bak")).expect("read backup"), + "keep this note\n" + ); +} + +#[test] +fn uninstall_codex_removes_only_managed_block() { + let codex_home = tempfile::tempdir().expect("temp codex home"); + let instructions_dir = codex_home.path().join("instructions"); + fs::create_dir(&instructions_dir).expect("create instructions dir"); + let path = instructions_dir.join("tinyjuice.md"); + fs::write( + &path, + "before\n\n\nCommand: /missing reduce-json\n\n\nafter\n", + ) + .expect("write managed instructions"); + + let output = tinyjuice() + .args([ + "uninstall", + "codex", + "--codex-home", + &codex_home.path().to_string_lossy(), + ]) + .output() + .expect("run tinyjuice uninstall codex"); + + assert!(output.status.success(), "{output:#?}"); + let response: serde_json::Value = + serde_json::from_slice(&output.stdout).expect("uninstall json"); + assert_eq!(response["changed"], true); + let updated = fs::read_to_string(&path).expect("read updated instructions"); + assert!(updated.contains("before"), "{updated}"); + assert!(updated.contains("after"), "{updated}"); + assert!(!updated.contains("tinyjuice-managed"), "{updated}"); +} From d7fcfeb366c920ffac9fd75ac2350c3981ecdf48 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Mon, 6 Jul 2026 10:34:32 +0000 Subject: [PATCH 102/120] Add compressor fixture coverage --- plan/content-compressor-roadmap.md | 12 + tests/compressor_fixtures.rs | 268 ++++++++++++++++++ .../diff_lockfile.fixture.json | 21 ++ .../json_query_anchor.fixture.json | 24 ++ .../search_ranked.fixture.json | 22 ++ .../textcrusher_signal.fixture.json | 23 ++ 6 files changed, 370 insertions(+) create mode 100644 tests/compressor_fixtures.rs create mode 100644 tests/compressor_fixtures/diff_lockfile.fixture.json create mode 100644 tests/compressor_fixtures/json_query_anchor.fixture.json create mode 100644 tests/compressor_fixtures/search_ranked.fixture.json create mode 100644 tests/compressor_fixtures/textcrusher_signal.fixture.json diff --git a/plan/content-compressor-roadmap.md b/plan/content-compressor-roadmap.md index fdc0161..2439875 100644 --- a/plan/content-compressor-roadmap.md +++ b/plan/content-compressor-roadmap.md @@ -39,6 +39,9 @@ Add: - Broader fixture coverage for all SmartCrusher planner paths. +Status: initial router-level fixture coverage now includes a query-anchored +row-dropping case with CCR recovery. More planner-path fixtures remain. + Do not add: - Non-deterministic learned planning in core. @@ -67,6 +70,9 @@ Add: - Broader fixture coverage for host-tuned noisy path policy. +Status: initial router-level fixture coverage now includes a lockfile noise +case that asserts the reason marker and recovery token. + Do not add: - Whitespace-only dropping by default without fixtures. @@ -162,6 +168,9 @@ Add: - Better query context propagation. - Broader fixture coverage for search-read host adapters. +Status: initial router-level fixture coverage now includes a query-ranked +search thinning case with omitted counts and CCR recovery. + Do not add: - Filesystem search in core compressor. @@ -220,6 +229,9 @@ Add: - Broader fixture coverage for TextCrusher scoring and ML tag protection. +Status: initial router-level fixture coverage now includes a query-relevant +error-preservation case with verbatim signal text and CCR recovery. + Do not add: - Abstractive summarization in core plain-text compressor. diff --git a/tests/compressor_fixtures.rs b/tests/compressor_fixtures.rs new file mode 100644 index 0000000..6b5bd2a --- /dev/null +++ b/tests/compressor_fixtures.rs @@ -0,0 +1,268 @@ +//! Fixture-driven coverage for router-level content compressors. +//! +//! The command-rule fixtures in `tests/fixtures/` cover reducer rules. These +//! fixtures cover non-command content compressors through the store-injected +//! router path that hosts use. + +use serde::Deserialize; +use std::fmt::Write as _; +use std::path::PathBuf; +use tinyjuice::cache::{CcrStore, MemoryCcrStore}; +use tinyjuice::{ + CompressOptions, CompressorKind, ContentHint, ContentKind, compress_content_with_store_report, +}; + +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +struct CompressorFixture { + description: String, + kind: String, + #[serde(default)] + extension: Option, + #[serde(default)] + query: Option, + generator: FixtureGenerator, + expected: FixtureExpectations, +} + +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase", tag = "type")] +enum FixtureGenerator { + JsonRows { + #[serde(rename = "rows")] + rows: usize, + #[serde(default)] + #[serde(rename = "specialRow")] + special_row: Option, + #[serde(default)] + #[serde(rename = "specialNote")] + special_note: Option, + }, + DiffLockfile { + #[serde(rename = "lines")] + lines: usize, + }, + SearchResults { + #[serde(rename = "matchesPerFile")] + matches_per_file: usize, + }, + PlainStatus { + #[serde(rename = "before")] + before: usize, + #[serde(rename = "after")] + after: usize, + #[serde(rename = "errorLine")] + error_line: String, + }, +} + +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +struct FixtureExpectations { + compressor: String, + lossy: bool, + recovery: bool, + #[serde(default)] + contains: Vec, + #[serde(default)] + not_contains: Vec, +} + +#[tokio::test] +async fn content_compressor_fixtures_match_expectations() { + for (path, fixture) in load_compressor_fixtures() { + let input = fixture.generator.render(); + let hint = ContentHint { + extension: fixture.extension.clone(), + query: fixture.query.clone(), + explicit: Some(parse_content_kind(&fixture.kind, &path)), + ..ContentHint::default() + }; + let store = MemoryCcrStore::default(); + let (out, report) = + compress_content_with_store_report(&input, Some(hint), &fixture_options(), &store) + .await; + + let expected_compressor = parse_compressor_kind(&fixture.expected.compressor, &path); + assert_eq!( + out.compressor, + expected_compressor, + "{}: {} produced report {report:?}", + path.display(), + fixture.description + ); + assert_eq!( + out.lossy, + fixture.expected.lossy, + "{}: unexpected lossiness for {}", + path.display(), + fixture.description + ); + assert_eq!( + out.ccr_token.is_some(), + fixture.expected.recovery, + "{}: unexpected recovery token state for {}", + path.display(), + fixture.description + ); + + for needle in &fixture.expected.contains { + assert!( + out.text.contains(needle), + "{}: expected output to contain {needle:?}\n---\n{}\n---", + path.display(), + out.text + ); + } + for needle in &fixture.expected.not_contains { + assert!( + !out.text.contains(needle), + "{}: expected output not to contain {needle:?}\n---\n{}\n---", + path.display(), + out.text + ); + } + + if fixture.expected.recovery { + let token = out.ccr_token.as_deref().expect("checked above"); + assert_eq!( + store.get(token).as_deref(), + Some(input.as_str()), + "{}: recovery token did not retrieve original input", + path.display() + ); + } + } +} + +fn fixture_options() -> CompressOptions { + CompressOptions { + min_bytes_to_compress: 0, + ccr_min_tokens: 0, + ..CompressOptions::default() + } +} + +fn load_compressor_fixtures() -> Vec<(PathBuf, CompressorFixture)> { + let dir = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("tests/compressor_fixtures"); + let mut paths: Vec<_> = std::fs::read_dir(&dir) + .unwrap_or_else(|error| panic!("cannot read {}: {error}", dir.display())) + .filter_map(|entry| entry.ok().map(|entry| entry.path())) + .filter(|path| { + path.file_name() + .and_then(|name| name.to_str()) + .is_some_and(|name| name.ends_with(".fixture.json")) + }) + .collect(); + paths.sort(); + assert!( + !paths.is_empty(), + "no compressor fixtures found in {}", + dir.display() + ); + + paths + .into_iter() + .map(|path| { + let raw = std::fs::read_to_string(&path) + .unwrap_or_else(|error| panic!("cannot read {}: {error}", path.display())); + let fixture = serde_json::from_str(&raw) + .unwrap_or_else(|error| panic!("invalid fixture {}: {error}", path.display())); + (path, fixture) + }) + .collect() +} + +fn parse_content_kind(kind: &str, path: &std::path::Path) -> ContentKind { + match kind { + "json" => ContentKind::Json, + "diff" => ContentKind::Diff, + "search" => ContentKind::Search, + "plainText" => ContentKind::PlainText, + other => panic!("{}: unknown content kind {other}", path.display()), + } +} + +fn parse_compressor_kind(kind: &str, path: &std::path::Path) -> CompressorKind { + match kind { + "smartcrusher" => CompressorKind::SmartCrusher, + "diff" => CompressorKind::Diff, + "search" => CompressorKind::Search, + "textcrusher" => CompressorKind::TextCrusher, + other => panic!("{}: unknown compressor kind {other}", path.display()), + } +} + +impl FixtureGenerator { + fn render(&self) -> String { + match self { + FixtureGenerator::JsonRows { + rows, + special_row, + special_note, + } => { + let rendered_rows: Vec<_> = (0..*rows) + .map(|i| { + let note = if Some(i) == *special_row { + special_note.as_deref().unwrap_or("special row") + } else { + "ordinary row" + }; + format!( + r#"{{"id":{i},"name":"record {i}","status":"active","note":"{note}"}}"# + ) + }) + .collect(); + format!("[{}]", rendered_rows.join(",")) + } + FixtureGenerator::DiffLockfile { lines } => { + let mut out = + String::from("diff --git a/Cargo.lock b/Cargo.lock\n@@ -1,80 +1,80 @@\n"); + for i in 0..*lines { + let _ = writeln!(out, "+ new dep entry {i}"); + } + for i in 0..*lines { + let _ = writeln!(out, "- old dep entry {i}"); + } + out + } + FixtureGenerator::SearchResults { matches_per_file } => { + let mut out = format!("{} match(es); scanned 2 file(s)\n", matches_per_file * 2); + for i in 0..*matches_per_file { + let body = if i == 7 { + "let needle_value = compute_long_name_7();".to_owned() + } else { + format!("let value_{i} = compute_long_name_{i}();") + }; + let _ = writeln!(out, "src/a.rs:{i}:{body}"); + } + for i in 0..*matches_per_file { + let _ = writeln!(out, "src/b.rs:{i}:fn helper_function_number_{i}() {{}}"); + } + out + } + FixtureGenerator::PlainStatus { + before, + after, + error_line, + } => { + let mut out = String::new(); + for i in 0..*before { + let _ = writeln!( + out, + "ordinary deployment progress line {i} with routine status information.\n" + ); + } + let _ = writeln!(out, "{error_line}\n"); + for i in 0..*after { + let _ = writeln!( + out, + "ordinary deployment progress line {} with routine status information.\n", + before + i + ); + } + out + } + } + } +} diff --git a/tests/compressor_fixtures/diff_lockfile.fixture.json b/tests/compressor_fixtures/diff_lockfile.fixture.json new file mode 100644 index 0000000..0e01ff2 --- /dev/null +++ b/tests/compressor_fixtures/diff_lockfile.fixture.json @@ -0,0 +1,21 @@ +{ + "description": "DiffNoise collapses generated lockfile body with a clear reason and retained recovery", + "kind": "diff", + "extension": "diff", + "generator": { + "type": "diffLockfile", + "lines": 80 + }, + "expected": { + "compressor": "diff", + "lossy": true, + "recovery": true, + "contains": [ + "reason=lockfile", + "tokenjuice_retrieve" + ], + "notContains": [ + "new dep entry 79" + ] + } +} diff --git a/tests/compressor_fixtures/json_query_anchor.fixture.json b/tests/compressor_fixtures/json_query_anchor.fixture.json new file mode 100644 index 0000000..9cbba64 --- /dev/null +++ b/tests/compressor_fixtures/json_query_anchor.fixture.json @@ -0,0 +1,24 @@ +{ + "description": "SmartCrusher row dropping keeps query-relevant rows and retains the original", + "kind": "json", + "extension": "json", + "query": "special needle", + "generator": { + "type": "jsonRows", + "rows": 140, + "specialRow": 71, + "specialNote": "contains special needle token" + }, + "expected": { + "compressor": "smartcrusher", + "lossy": true, + "recovery": true, + "contains": [ + "special needle token", + "tokenjuice_retrieve" + ], + "notContains": [ + "\"id\": 70" + ] + } +} diff --git a/tests/compressor_fixtures/search_ranked.fixture.json b/tests/compressor_fixtures/search_ranked.fixture.json new file mode 100644 index 0000000..3b9ef8a --- /dev/null +++ b/tests/compressor_fixtures/search_ranked.fixture.json @@ -0,0 +1,22 @@ +{ + "description": "Search thinning preserves query-ranked match, omitted counts, and recovery", + "kind": "search", + "query": "needle_value", + "generator": { + "type": "searchResults", + "matchesPerFile": 60 + }, + "expected": { + "compressor": "search", + "lossy": true, + "recovery": true, + "contains": [ + "needle_value", + "search omitted", + "tokenjuice_retrieve" + ], + "notContains": [ + "helper_function_number_59" + ] + } +} diff --git a/tests/compressor_fixtures/textcrusher_signal.fixture.json b/tests/compressor_fixtures/textcrusher_signal.fixture.json new file mode 100644 index 0000000..6a6180d --- /dev/null +++ b/tests/compressor_fixtures/textcrusher_signal.fixture.json @@ -0,0 +1,23 @@ +{ + "description": "TextCrusher keeps query-relevant error text verbatim and retains recovery", + "kind": "plainText", + "query": "sync.worker.v2 REQUEST_ID", + "generator": { + "type": "plainStatus", + "before": 40, + "after": 40, + "errorLine": "ERROR sync.worker.v2 failed for REQUEST_ID 9F42 after retry 17." + }, + "expected": { + "compressor": "textcrusher", + "lossy": true, + "recovery": true, + "contains": [ + "ERROR sync.worker.v2 failed for REQUEST_ID 9F42", + "tokenjuice_retrieve" + ], + "notContains": [ + "ordinary deployment progress line 3 with" + ] + } +} From ee726aaac5ce126b0ce5ca229aa9b07ce84ccdd4 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Mon, 6 Jul 2026 10:36:42 +0000 Subject: [PATCH 103/120] Add web extract fixture coverage --- plan/content-compressor-roadmap.md | 4 + tests/web_extract_fixtures.rs | 201 ++++++++++++++++++ .../markdown_article.fixture.json | 40 ++++ 3 files changed, 245 insertions(+) create mode 100644 tests/web_extract_fixtures.rs create mode 100644 tests/web_extract_fixtures/markdown_article.fixture.json diff --git a/plan/content-compressor-roadmap.md b/plan/content-compressor-roadmap.md index 2439875..74ff3f9 100644 --- a/plan/content-compressor-roadmap.md +++ b/plan/content-compressor-roadmap.md @@ -199,6 +199,10 @@ Add: - More complete entity support if fixtures show need. - Broader fixture coverage for web-extract host formats. +Status: initial web-extract fixture coverage now includes a markdown host +format case for base64 image replacement, real image URL preservation, URL +secret redaction, and retained recovery text. + Do not add: - URL fetching or scraping in core TinyJuice. diff --git a/tests/web_extract_fixtures.rs b/tests/web_extract_fixtures.rs new file mode 100644 index 0000000..5139ae5 --- /dev/null +++ b/tests/web_extract_fixtures.rs @@ -0,0 +1,201 @@ +//! Fixture-driven coverage for already-extracted web page reduction. + +use serde::Deserialize; +use serde_json::Map; +use std::fmt::Write as _; +use std::path::PathBuf; +use tinyjuice::cache::{CcrStore, MemoryCcrStore}; +use tinyjuice::{ + WebExtractFormat, WebExtractOptions, WebExtractReduceInput, reduce_web_extract_with_store, +}; + +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +struct WebExtractFixture { + description: String, + url: String, + #[serde(default)] + title: Option, + #[serde(default)] + format: WebExtractFormat, + char_limit: usize, + generator: WebExtractGenerator, + expected: WebExtractExpectations, +} + +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase", tag = "type")] +enum WebExtractGenerator { + MarkdownArticle { + #[serde(rename = "paragraphs")] + paragraphs: usize, + #[serde(rename = "base64Payload")] + base64_payload: String, + #[serde(rename = "remoteImageUrl")] + remote_image_url: String, + }, +} + +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +struct WebExtractExpectations { + truncated: bool, + full_text_retained: bool, + base64_images_replaced: usize, + #[serde(default)] + contains: Vec, + #[serde(default)] + not_contains: Vec, + #[serde(default)] + serialized_not_contains: Vec, + recovered_text_contains: Vec, + recovered_text_not_contains: Vec, +} + +#[test] +fn web_extract_fixtures_match_expectations() { + for (path, fixture) in load_web_extract_fixtures() { + let input = WebExtractReduceInput { + url: fixture.url, + title: fixture.title, + content: fixture.generator.render(), + format: fixture.format, + provider: Some("fixture".to_owned()), + char_limit: Some(fixture.char_limit), + metadata: Map::new(), + }; + let store = MemoryCcrStore::new(10, input.content.len() * 2); + let out = reduce_web_extract_with_store(&input, &fixture_options(), &store); + + assert_eq!( + out.truncated, + fixture.expected.truncated, + "{}: unexpected truncation state for {}", + path.display(), + fixture.description + ); + assert_eq!( + out.full_text_retained, + fixture.expected.full_text_retained, + "{}: unexpected retention state for {}", + path.display(), + fixture.description + ); + assert_eq!( + out.base64_images_replaced, + fixture.expected.base64_images_replaced, + "{}: unexpected base64 replacement count", + path.display() + ); + + for needle in &fixture.expected.contains { + assert!( + out.text.contains(needle), + "{}: output missing {needle:?}\n---\n{}\n---", + path.display(), + out.text + ); + } + for needle in &fixture.expected.not_contains { + assert!( + !out.text.contains(needle), + "{}: output unexpectedly contained {needle:?}\n---\n{}\n---", + path.display(), + out.text + ); + } + + let serialized = serde_json::to_string(&out).expect("serialize web extract reduction"); + for needle in &fixture.expected.serialized_not_contains { + assert!( + !serialized.contains(needle), + "{}: serialized metadata unexpectedly contained {needle:?}", + path.display() + ); + } + + if fixture.expected.full_text_retained { + let token = out.ccr_token.as_deref().expect("retained token"); + let recovered = store.get(token).expect("recover retained web text"); + for needle in &fixture.expected.recovered_text_contains { + assert!( + recovered.contains(needle), + "{}: recovered text missing {needle:?}", + path.display() + ); + } + for needle in &fixture.expected.recovered_text_not_contains { + assert!( + !recovered.contains(needle), + "{}: recovered text unexpectedly contained {needle:?}", + path.display() + ); + } + } + } +} + +fn fixture_options() -> WebExtractOptions { + WebExtractOptions { + char_limit: 120, + min_char_limit: 40, + max_char_limit: 500, + head_ratio: 0.75, + convert_base64_images: true, + max_combined_inline_chars: 1000, + } +} + +fn load_web_extract_fixtures() -> Vec<(PathBuf, WebExtractFixture)> { + let dir = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("tests/web_extract_fixtures"); + let mut paths: Vec<_> = std::fs::read_dir(&dir) + .unwrap_or_else(|error| panic!("cannot read {}: {error}", dir.display())) + .filter_map(|entry| entry.ok().map(|entry| entry.path())) + .filter(|path| { + path.file_name() + .and_then(|name| name.to_str()) + .is_some_and(|name| name.ends_with(".fixture.json")) + }) + .collect(); + paths.sort(); + assert!( + !paths.is_empty(), + "no web extract fixtures found in {}", + dir.display() + ); + + paths + .into_iter() + .map(|path| { + let raw = std::fs::read_to_string(&path) + .unwrap_or_else(|error| panic!("cannot read {}: {error}", path.display())); + let fixture = serde_json::from_str(&raw) + .unwrap_or_else(|error| panic!("invalid fixture {}: {error}", path.display())); + (path, fixture) + }) + .collect() +} + +impl WebExtractGenerator { + fn render(&self) -> String { + match self { + WebExtractGenerator::MarkdownArticle { + paragraphs, + base64_payload, + remote_image_url, + } => { + let mut out = format!( + "intro ![inline](data:image/png;base64,{base64_payload}) ![remote]({remote_image_url})\n" + ); + for i in 0..*paragraphs { + let _ = writeln!( + out, + "paragraph-{i:03}: extracted article text with details and repeated context." + ); + } + out.push_str("final web sentinel\n"); + out + } + } + } +} diff --git a/tests/web_extract_fixtures/markdown_article.fixture.json b/tests/web_extract_fixtures/markdown_article.fixture.json new file mode 100644 index 0000000..485cbd2 --- /dev/null +++ b/tests/web_extract_fixtures/markdown_article.fixture.json @@ -0,0 +1,40 @@ +{ + "description": "Markdown web extraction strips inline base64, keeps real image URLs, redacts URL secrets, and retains recovery", + "url": "https://example.com/article/path?token=secret&session=private", + "title": "Fixture Article", + "format": "markdown", + "charLimit": 120, + "generator": { + "type": "markdownArticle", + "paragraphs": 30, + "base64Payload": "AAAA1111SECRETIMAGE", + "remoteImageUrl": "https://cdn.example/images/chart.png" + }, + "expected": { + "truncated": true, + "fullTextRetained": true, + "base64ImagesReplaced": 1, + "contains": [ + "[IMAGE: inline]", + "https://cdn.example/images/chart.png", + "final web sentinel", + "tokenjuice_retrieve" + ], + "notContains": [ + "AAAA1111SECRETIMAGE" + ], + "serializedNotContains": [ + "token=secret", + "session=private", + "/article/path" + ], + "recoveredTextContains": [ + "[IMAGE: inline]", + "https://cdn.example/images/chart.png", + "paragraph-029" + ], + "recoveredTextNotContains": [ + "AAAA1111SECRETIMAGE" + ] + } +} From 706ecd320793800e1ac172a78ac6867d0bbaf615 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Mon, 6 Jul 2026 10:39:08 +0000 Subject: [PATCH 104/120] Add hook-path compressor e2e coverage --- plan/openhuman-algorithm-port-plan.md | 6 ++ tests/e2e_tool_output.rs | 149 ++++++++++++++++++++++++++ 2 files changed, 155 insertions(+) diff --git a/plan/openhuman-algorithm-port-plan.md b/plan/openhuman-algorithm-port-plan.md index 59f93c0..9de0ce7 100644 --- a/plan/openhuman-algorithm-port-plan.md +++ b/plan/openhuman-algorithm-port-plan.md @@ -326,6 +326,12 @@ Acceptance: as specified per compressor in the roadmap plan; plus one OpenHuman end-to-end fixture per compressor family driven through `compact_tool_output_with_policy`. +Status: TinyJuice now has host-hook e2e coverage through +`compact_tool_output_with_policy` for SmartCrusher, DiffNoise, search +thinning/BM25 query ranking, TextCrusher, HTML extraction, and generic command +fallback, each asserting recoverability where the path is lossy. OpenHuman-side +adapter fixtures remain the host-repo portion of this acceptance item. + ## P2-1: Ranked Search-Read Tool Source spec: `ranked-search-read-spec.md`. This is a NEW host tool, not a diff --git a/tests/e2e_tool_output.rs b/tests/e2e_tool_output.rs index dbf2d54..e53e5ba 100644 --- a/tests/e2e_tool_output.rs +++ b/tests/e2e_tool_output.rs @@ -9,6 +9,7 @@ mod common; use serde_json::json; +use std::fmt::Write as _; use tinyjuice::cache; use tinyjuice::tool_integration::compact_tool_output_with_policy; use tinyjuice::types::AgentTokenjuiceCompression; @@ -31,6 +32,72 @@ fn big_json_rows() -> String { serde_json::to_string_pretty(&rows).unwrap() } +fn lockfile_diff() -> String { + let mut s = String::from("diff --git a/Cargo.lock b/Cargo.lock\n@@ -1,80 +1,80 @@\n"); + for i in 0..80 { + let _ = writeln!(s, "+ new dep entry {i}"); + } + for i in 0..80 { + let _ = writeln!(s, "- old dep entry {i}"); + } + s +} + +fn search_results() -> String { + let mut s = String::from("120 match(es); scanned 2 file(s)\n"); + for i in 0..60 { + let body = if i == 7 { + "let needle_value = compute_long_name_7();".to_owned() + } else { + format!("let value_{i} = compute_long_name_{i}();") + }; + let _ = writeln!(s, "src/a.rs:{i}:{body}"); + } + for i in 0..60 { + let _ = writeln!(s, "src/b.rs:{i}:fn helper_function_number_{i}() {{}}"); + } + s +} + +fn status_report_text() -> String { + let mut s = String::new(); + for i in 0..40 { + let _ = writeln!( + s, + "ordinary deployment progress line {i} with routine status information.\n" + ); + } + s.push_str("ERROR sync.worker.v2 failed for REQUEST_ID 9F42 after retry 17.\n\n"); + for i in 40..80 { + let _ = writeln!( + s, + "ordinary deployment progress line {i} with routine status information.\n" + ); + } + s +} + +fn html_document() -> String { + let mut body = String::from( + "Fixture

Fixture page

", + ); + for i in 0..120 { + let _ = write!( + body, + "

paragraph {i} with useful extracted text and repeated markup wrappers.

" + ); + } + body.push_str(""); + body +} + +fn assert_recoverable(text: &str, expected_original: &str) { + let tokens = cache::parse_markers(text); + assert_eq!(tokens.len(), 1, "expected one recovery marker in:\n{text}"); + let original = cache::retrieve(&tokens[0]).expect("footer token must be retrievable"); + assert_eq!(original, expected_original); +} + #[tokio::test] async fn off_profile_is_exact_passthrough() { common::install_test_config(); @@ -121,6 +188,88 @@ async fn full_profile_compacts_and_original_is_recoverable() { assert_eq!(original, payload, "CCR roundtrip must restore the original"); } +#[tokio::test] +async fn diff_noise_compacts_recoverably_through_tool_hook() { + common::install_test_config(); + let payload = lockfile_diff(); + let (text, stats) = compact_tool_output_with_policy( + "shell", + Some(&json!({ "command": "git diff Cargo.lock" })), + &payload, + Some(0), + AgentTokenjuiceCompression::Full, + ) + .await; + + assert!(stats.applied, "expected DiffNoise, got {stats:?}"); + assert_eq!(stats.rule_id, "diff"); + assert!(text.contains("reason=lockfile"), "{text}"); + assert!(stats.compacted_bytes < stats.original_bytes); + assert_recoverable(&text, &payload); +} + +#[tokio::test] +async fn search_results_compact_recoverably_through_tool_hook() { + common::install_test_config(); + let payload = search_results(); + let (text, stats) = compact_tool_output_with_policy( + "grep", + Some(&json!({ "pattern": "needle_value" })), + &payload, + Some(0), + AgentTokenjuiceCompression::Full, + ) + .await; + + assert!(stats.applied, "expected search compressor, got {stats:?}"); + assert_eq!(stats.rule_id, "search"); + assert!(text.contains("needle_value"), "{text}"); + assert!(text.contains("search omitted"), "{text}"); + assert!(stats.compacted_bytes < stats.original_bytes); + assert_recoverable(&text, &payload); +} + +#[tokio::test] +async fn textcrusher_compacts_recoverably_through_tool_hook() { + common::install_test_config(); + let payload = status_report_text(); + let (text, stats) = compact_tool_output_with_policy( + "browser_extract_text", + Some(&json!({ "query": "sync.worker.v2 REQUEST_ID" })), + &payload, + Some(0), + AgentTokenjuiceCompression::Full, + ) + .await; + + assert!(stats.applied, "expected TextCrusher, got {stats:?}"); + assert_eq!(stats.rule_id, "textcrusher"); + assert!(text.contains("ERROR sync.worker.v2 failed"), "{text}"); + assert!(stats.compacted_bytes < stats.original_bytes); + assert_recoverable(&text, &payload); +} + +#[tokio::test] +async fn html_extract_compacts_recoverably_through_tool_hook() { + common::install_test_config(); + let payload = html_document(); + let (text, stats) = compact_tool_output_with_policy( + "http_request", + Some(&json!({ "url": "https://example.test/page", "path": "page.html" })), + &payload, + Some(0), + AgentTokenjuiceCompression::Full, + ) + .await; + + assert!(stats.applied, "expected HTML extraction, got {stats:?}"); + assert_eq!(stats.rule_id, "html"); + assert!(text.contains("Fixture page"), "{text}"); + assert!(!text.contains("