diff --git a/Cargo.toml b/Cargo.toml index e8a0cc6..bf0e118 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,6 +8,7 @@ repository = "https://github.com/tinyhumansai/tinyjuice" readme = "README.md" keywords = ["llm", "tokens", "compression", "openhuman", "context"] categories = ["text-processing"] +autobins = false [dependencies] async-trait = "0.1" @@ -49,6 +50,14 @@ tokio = { version = "1", features = ["macros", "rt", "test-util"] } name = "compression" harness = false +[[bin]] +name = "tinyjuice" +path = "src/main.rs" + +[[bin]] +name = "tinyjuice-hooks" +path = "src/bin/tinyjuice.rs" + [features] default = ["tinyjuice-treesitter"] tinyjuice-treesitter = [ diff --git a/README.md b/README.md index 1f1e1da..20578e6 100644 --- a/README.md +++ b/README.md @@ -26,6 +26,146 @@ behind a retrieval token so it can be pulled back on demand. Hosts that want the strict lossless-or-recoverable guarantee (e.g. coding agents on the `light` profile) can require a recovery token for any lossy output. +## Highlights + +- **Content-aware by default** - JSON, code, logs, search results, diffs, HTML, + and plain text take different paths instead of one generic truncation rule. +- **Recoverable lossy views** - the CCR cache stores exact originals and appends + a `tokenjuice_retrieve` footer whenever data is dropped. +- **Agent-profile policy** - hosts can run `full`, `light`, `off`, or runtime + `auto` profiles per agent instead of using one global behavior. +- **Command-aware reduction** - built-in rules compact common shell, git, cargo, + npm, docker, kubectl, database, cloud, lint, and test outputs. +- **OpenHuman-ready boundary** - the core crate avoids OpenHuman runtime + dependencies; adapters install configuration, ML callbacks, and savings + recorders from the host side. +- **No raw-content analytics requirement** - the dashboard consumes metadata, + token and byte counts, latency, status, and strategy labels, not prompt text. + +TinyJuice is designed for the work agents actually do: reading too much, +searching broadly, running noisy commands, and needing a compact but reversible +view that keeps failures, anomalies, changed hunks, signatures, and matching +lines visible. + +## How It Works + +```text +tool output / file / web payload + | + v +ContentHint + structural detection + | + v +JSON | Code | Log | Search | Diff | HTML | PlainText + | + v +specialized compressor or command-rule reducer + | + v +pass-through if unsafe, too small, disabled, or not smaller + | + v +CCR offload + retrieval footer when the view is lossy +``` + +The router is intentionally fail-soft. If it cannot shrink safely, it returns +the original bytes unchanged. + +## Compression Surfaces + +- **JSON SmartCrusher** - renders repeated object arrays as compact tables, + flattens safe nested cells, and keeps query-relevant, query-direction, + anomaly, numeric change-point, information-dense, duplicate/near-duplicate + cluster, and saturation/knee-based spread-anchor rows when large arrays are + row-dropped. +- **Code compressor** - keeps imports, signatures, shallow structure, and + important markers while collapsing deep bodies. +- **Log compressor** - preserves failures, warnings, summaries, stack traces, + command-rule outputs, and reconstructible high-context template runs while + dropping passing noise. +- **Search compressor** - groups grep/ripgrep output by file, ranks matches + with shared BM25 query context, and keeps top hits with per-file and global + omitted-match tallies. +- **Diff compressor** - keeps patch structure and changed lines, collapses long + context, and marks omitted lockfile, generated-bundle, or configured noisy + hunks with explicit reasons. +- **HTML compressor** - extracts readable text from rendered markup. +- **Plain-text ML slot** - optional host-provided callback for learned text + compression; disabled by default. +- **Generic command fallback** - line-oriented head/tail reduction for command + output when no specialized rule wins. + +TinyJuice does not publish compression percentage claims yet. Throughput +benchmarks exist for hot paths, but ratio and quality claims require benchmark +fixtures that prove retained facts, latency, reversibility, and regression +safety. + +## Quick Start + +Add TinyJuice to a Rust project once published: + +```toml +[dependencies] +tinyjuice = "0.1" +``` + +Use the small public trait scaffold when you want a simple strategy boundary: + +```rust +use tinyjuice::{CompressionConfig, CompressionInput, Compressor, PassthroughCompressor}; + +fn main() -> Result<(), tinyjuice::TinyJuiceError> { + let compressor = PassthroughCompressor; + let output = compressor.compress( + CompressionInput::new("Keep this text unchanged for now."), + &CompressionConfig::default(), + )?; + + assert_eq!(output.report.strategy, "passthrough"); + Ok(()) +} +``` + +Use the content router for real tool-output compaction: + +```rust +use tinyjuice::{CompressOptions, ContentHint, compress_content}; + +async fn compact_payload(big_payload: &str) { + let hint = ContentHint { + source_tool: Some("read_file".to_string()), + extension: Some("json".to_string()), + ..Default::default() + }; + + let result = compress_content(big_payload, Some(hint), &CompressOptions::default()).await; + if result.applied { + println!("{} -> {} bytes", result.original_bytes, result.compacted_bytes); + } +} +``` + +OpenHuman-style tool output integration goes through: + +```rust +use tinyjuice::{AgentTokenjuiceCompression, compact_tool_output_with_policy}; + +async fn compact_command_output(command_output: &str) { + let (_text, _stats) = compact_tool_output_with_policy( + "shell", + Some(&serde_json::json!({ "command": "cargo test" })), + command_output, + Some(101), + AgentTokenjuiceCompression::Full, + ).await; +} +``` + +Host agent layers should resolve `AgentTokenjuiceCompression::Auto` to `Full`, +`Light`, or `Off` before calling TokenJuice. Passing unresolved `Auto` to the +tool-output adapter leaves the output unchanged and reports +`none/agent-profile-auto-unresolved`. + ## Quick Setup Install the CLI: @@ -34,6 +174,26 @@ Install the CLI: cargo install tinyjuice --locked ``` +Run the minimal reducer CLI: + +```sh +cargo run -- reduce --tool-name bash --command "git status" status.txt +cargo run -- reduce-json payload.json +cargo run -- verify --rules --fixtures +cargo run -- discover executions.ndjson +cargo run -- wrap -- cargo test +cargo run -- ls --store-dir .tokenjuice/ccr +cargo run -- cat --store-dir .tokenjuice/ccr +cargo run -- stats --store-dir .tokenjuice/ccr +cargo run -- doctor --store-dir .tokenjuice/ccr +cargo run -- doctor codex +cargo run -- doctor hooks +cargo run -- install codex --local target/debug/tinyjuice +cargo run -- uninstall codex +``` + +Run hot-path benchmarks: + Run one hook installer: | Logo | Client | Install | @@ -41,6 +201,110 @@ Run one hook installer: | Codex | [Codex CLI](https://github.com/openai/codex) | `tinyjuice install codex` | | Claude Code | [Claude Code](https://docs.anthropic.com/en/docs/claude-code) | `tinyjuice install claude-code` | +## API Notes + +`compress_content` and `route` return `CompressedOutput`. The `text` field is +the compatibility output ready to inline into model context. When CCR retained +an original, `body` contains the compacted body without the recovery footer and +`recovery_footer` contains the footer separately. Hosts that apply their own +post-compaction caps should truncate only `body` and then reattach +`recovery_footer` so recovery markers remain reachable. + +`compress_content_with_store` and `route_with_store` accept an injectable +`CcrStore`. Their `_report` variants also return a redacted `PipelineReport` +with byte counts, a cheap bloat estimate, applied steps, skip reason, lossy +flag, and CCR token IDs. Existing `compress_content`, `route`, and `cache::*` +helpers still use the process-global `GlobalCcrStore` for compatibility. New +tests and host adapters can use `MemoryCcrStore` to assert CCR behavior without +touching the global cache. + +The CLI `ls`, `cat`, and `stats` commands operate only on an explicit CCR disk +tier via `--store-dir`; they do not imply access to another process's in-memory +cache. `cat` supports `--lines START:END` and `--bytes START:END` ranges. +`stats` reports metadata-only counts and byte totals without reading token +content. + +The CLI `doctor` command emits structured health JSON using `ok`, `warn`, +`broken`, and `disabled` statuses. It verifies built-in rule health by default, +can include fixture checks with `--fixtures [dir]`, and can inspect an explicit +CCR disk tier with `--store-dir`. +Host-aware doctor targets (`codex`, `openhuman`, and aggregate `hooks`) expose +expected commands, detected commands when present, and one repair command +without mutating user configuration. +The first host mutation path is `install codex` / `uninstall codex`, which +maintains only a TinyJuice-managed instruction block under the Codex config +root and preserves unrelated text. + +`run_typed_pipeline` is the typed-transform entry point for new reducers. It +runs lossless `ReformatTransform`s before CCR-backed `OffloadTransform`s and +keeps the true original content available to offload transforms even after an +intermediate reformat. + +Shell-producing hosts can use the `route*_shell_policy` variants or call +`apply_shell_compaction_policy` with `ShellCompactionPolicy` before compacting +command output. The default OpenHuman-facing path uses `AllowSafeInventory`: +exact file-content reads, unsafe inventory actions, and mixed shell sequences +stay raw, while repository inventory output can still be summarized. + +Conversation-level helpers under `conversation::*` expose deterministic Hermes +primitives without host runtime dependencies: token-budget tail selection, +tool-call/result boundary alignment, latest user/assistant anchors, JSON +string-leaf shrinking, partial split/rejoin helpers for compacting a middle +window, last-N user exchange retention helpers, and old tool-result digesting +with sensitive metadata redaction. + +`savings::configure_record_recorder` installs a metadata-only savings recorder +that receives class-labeled `SavingsRecord` values (`counted`, `measured`, or +`estimated`) with a source label (`live` or `fixture_benchmark`), content kind, +compressor, byte/token counts, lossy/CCR flags, and redacted rule or skip +labels. Fixture benchmark records are meant to be reported separately from live +runtime stats. The older four-argument +`configure_recorder` callback remains as a compatibility wrapper. + +`ContextBreakdown` and `ContextBucket` provide host-facing context usage +metadata for UI bars and compression diagnostics. Buckets separate static +prefix costs such as system prompts and tool definitions from compressible +conversation or memory history, and measured provider prompt usage can override +rough local estimates without storing raw prompt text. + +`live_zone::*` exposes provider-neutral cache/live-zone contracts for hosts +that need to preserve frozen prompt bytes exactly. Hosts provide byte ranges; +TinyJuice validates mutable-block replacements, preserves the frozen prefix, +and can detect volatile cache-hostile values such as UUIDs, timestamps, JWTs, +and hex hashes with redacted findings only. + +`reduce_json_str` and `reduce_json_request` expose the library form of the +`reduce-json` protocol. They accept direct `ToolExecutionInput` JSON or an +`{ "input": ..., "options": ... }` envelope, reject malformed payloads with +structured errors, and return stable serde-compatible response fields. See +[docs/reduce-json-protocol.md](docs/reduce-json-protocol.md) for the current +request and response contract. + +`verify_rules` checks the same builtin/user/project rule layers as the loader +without changing the lenient load contract. It reports parse errors, invalid +regex patterns, duplicate rule IDs, and shadowed lower-priority rules so CLI +or host diagnostics can fail loudly while runtime loading remains compatible. +`verify_rule_fixtures` runs recorded `*.fixture.json` examples through compiled +rules and reports pass counts, parse errors, and hash-only output mismatches. +`discover_fallback_outputs` groups command families that still fall through to +`generic/fallback` without including raw tool output in the report. + +Already-extracted web pages can be passed through `reduce_web_extract` or +`reduce_web_extract_with_store`. The reducer removes inline base64 image blobs, +preserves ordinary markdown image URLs, and stores omitted middle content in CCR +before emitting a head/tail truncation footer. If CCR cannot retain the full +cleaned page, the reducer returns the cleaned page without lossy truncation. + +Source-code file reads are exact by default. Hosts that intentionally want a +structural view can call `stub_code` with a `StubMode`; the returned +`CodeStubOutput` includes symbols, elided line ranges, and whether tree-sitter +or the heuristic fallback produced the stub. `PublicApi` stubs keep imports and +public signatures while replacing private declarations with elision metadata; +matched-symbol and line-range expansion also work through the heuristic +fallback. + +Run the local analytics interface: + Use `tinyjuice update ` to refresh an installed hook and `tinyjuice uninstall ` to remove it. @@ -64,6 +328,28 @@ attribution for agent-created commits. - **Privacy-aware by design** - analytics can use metadata, byte counts, latency, status, and strategy labels without requiring raw prompt text. +```text +src/ + cache/ CCR recovery stores and retrieval markers + compressor/ Compression trait and input/output types + compressors/ Content-aware compressor implementations + config/ Compression target and policy configuration + conversation/ Provider-neutral conversation compaction helpers + detect/ Content-kind hints and structural detection + observability.rs Non-sensitive context usage breakdowns + pipeline/ Typed transform/report primitives + policy/ Host compaction policy helpers + reduce/ Rule-engine reducers and command normalization + rules/ Built-in, user, and project command reduction rules + savings.rs Host-installed savings attribution hook + tool_integration.rs OpenHuman-style tool-output adapter + openhuman/ Placeholder OpenHuman integration boundary + error.rs Shared error type +interface/ Self-hostable analytics UI for compression run metadata +wiki/ Technical GitHub wiki source +docs/references/ Design references and candidate strategy specs +``` + ## What It Compresses | Surface | What stays visible | @@ -164,6 +450,15 @@ The technical docs live in the wiki: - [Development](wiki/Development.md) - [Security and Privacy](wiki/Security-and-Privacy.md) -TinyJuice is pre-1.0. The CLI, router, command-rule engine, CCR recovery store, -content detectors, native compressors, and OpenHuman-style adapter are in place; -public API names may still move as host integration hardens. +## Status + +TinyJuice is pre-1.0. The router, command-rule engine, CCR recovery store, +content detectors, several native compressors, the OpenHuman-style tool adapter, +typed-pipeline primitives, injectable CCR store, report-producing router path, +savings metadata, deterministic conversation helpers, live-zone cache contracts, +and the analytics interface are implemented. Public API names may still move as OpenHuman +integration hardens. + +The project boundary is deliberate: keep the core crate small, do not add +OpenHuman runtime dependencies without a feature or adapter boundary, and do not +claim compression percentages until benchmark fixtures exist. diff --git a/docs/reduce-json-protocol.md b/docs/reduce-json-protocol.md new file mode 100644 index 0000000..6195217 --- /dev/null +++ b/docs/reduce-json-protocol.md @@ -0,0 +1,151 @@ +# Reduce-JSON Protocol + +`reduce_json_str` and `reduce_json_request` provide the library form of the +`tinyjuice reduce-json` machine protocol. The current surface is meant for Rust +hosts, the CLI, and tests; artifact records and durable stats recording are +still product-roadmap work. + +## Request Shapes + +The protocol accepts either a direct `ToolExecutionInput` JSON object or an +envelope with `input` and `options`. + +Direct input: + +```json +{ + "toolName": "bash", + "argv": ["git", "status"], + "stdout": "On branch main\n\nChanges not staged for commit:\n\tmodified: src/lib.rs\n" +} +``` + +Envelope input: + +```json +{ + "input": { + "toolName": "bash", + "argv": ["cargo", "test"], + "exitCode": 101, + "stdout": "running 1 test\ntest api::works ... FAILED\n" + }, + "options": { + "maxInlineChars": 1200, + "trace": true, + "recordStats": true + } +} +``` + +`ToolExecutionInput` fields use camelCase. Common fields are: + +- `toolName`: host tool name such as `bash`, `shell`, or `exec`. +- `command` and `argv`: command identity used by rule classification and shell + policy. +- `stdout`, `stderr`, or `combinedText`: raw tool output. If `combinedText` is + present, it takes priority. +- `exitCode`: optional process status for failure-preserving reducers. +- `cwd`: optional project-rule lookup root. +- `args`, `metadata`, and timing fields: accepted for host context; raw values + are not echoed in trace output. + +## Options + +`options` is optional. Supported fields are: + +- `classifier`: force a reducer id, falling back to normal matching if invalid. +- `maxInlineChars`: cap returned inline text. +- `raw`: return raw text without reducing it. +- `noOmit`: compatibility flag recorded as metadata. +- `store`: when reduction omits content and CCR retains the original, return a + metadata-only CCR reference. +- `storeDir`: accepted for compatibility; explicit per-call artifact storage is + not wired in the library surface. +- `cwd`: copied to `input.cwd` when the input does not already set it. +- `trace`: include metadata-only classification trace fields. +- `recordStats`: emit a metadata-only `SavingsRecord` through the configured + savings recorder, if any. + +## Response + +Successful responses serialize as: + +```json +{ + "inlineText": "Changes not staged:\nM: src/lib.rs", + "stats": { + "rawChars": 71, + "reducedChars": 32, + "ratio": 0.4507042253521127 + }, + "classification": { + "family": "git-status", + "matchedReducer": "git/status", + "confidence": "high", + "reasons": ["argv0 matched"] + } +} +``` + +Optional fields: + +- `previewText`: reducer preview when a future surface supplies one. +- `facts`: counter results such as failed tests or errors. +- `metadata`: booleans for `noOmit`, `store`, and `recordStats` requests. + When `store` succeeds, `metadata.ccr` contains a `token` plus + `originalChars`. The token references the original raw tool output in CCR; + it is not embedded in `inlineText`. +- `trace`: metadata-only reducer trace with `toolName`, `argv0`, `rawMode`, + `maxInlineChars`, `family`, and `matchedReducer`. + +Trace output intentionally excludes raw command output, raw arguments, and full +metadata maps. + +`recordStats` uses the same metadata-only savings recorder as the content +router. Records include reducer id, measured character counts as byte-count +metadata for this protocol surface, estimated token counts, and CCR-token +presence. They do not include raw output, command arguments, or host metadata. + +Example `store` metadata: + +```json +{ + "metadata": { + "storeRequested": true, + "noOmitRequested": false, + "recordStatsRequested": false, + "ccr": { + "token": "0123456789abcdef0123456789abcdef", + "originalChars": 4096 + } + } +} +``` + +## Errors + +Malformed protocol payloads return structured errors instead of echoing input: + +```json +{ + "code": "invalid_json", + "message": "expected `,` or `}` at line 1 column 39" +} +``` + +Current error codes: + +- `invalid_json`: JSON parsing or request-shape decoding failed. +- `nul_byte`: the payload contained a NUL byte. + +## Safety Notes + +The protocol reducer uses the same built-in, user, and project rule layers as +the Rust reducer. Runtime rule loading remains lenient, while `verify_rules` +and `verify_rule_fixtures` are available for callers that want strict +diagnostics before accepting a rule set. + +Shell output uses the default safe-inventory policy. Exact file-content reads, +mixed shell sequences, and unsafe inventory actions pass through unchanged; +safe inventory output can still be summarized. diff --git a/docs/references/web-extract-truncate-store-spec.md b/docs/references/web-extract-truncate-store-spec.md index 50453fe..af4fa5d 100644 --- a/docs/references/web-extract-truncate-store-spec.md +++ b/docs/references/web-extract-truncate-store-spec.md @@ -2,8 +2,10 @@ ## Status -Design reference for web extraction output reduction. This is a specification -only; TinyJuice does not yet implement a web extraction adapter. +Implemented as an adapter-facing reducer in `src/compressors/web_extract.rs`. +TinyJuice still does not fetch URLs; hosts pass already extracted text and +metadata into the reducer. OpenHuman wires this through `web_fetch` and +`http_request`. ## Purpose @@ -14,7 +16,7 @@ path for these pages: - return small extracted pages whole; - replace inline base64 image blobs with placeholders; - return a bounded head/tail window for large pages; -- store the full extracted text in a recoverable artifact; +- store the full extracted text in CCR for model-facing recovery; - include an explicit retrieval footer in the model-facing text. This is not semantic summarization. It does not ask an auxiliary model to decide @@ -78,7 +80,6 @@ WebExtractBatchInput { pages: Vec, default_char_limit: usize, max_combined_inline_chars: usize, - store_policy: StorePolicy, } ``` @@ -93,7 +94,6 @@ char_limit = 15000 min_char_limit = 2000 max_char_limit = 500000 head_ratio = 0.75 -store_full_text = true convert_base64_images = true max_combined_inline_chars = 100000 ``` @@ -124,7 +124,6 @@ reduce_web_extract(page, options): if len(clean) <= limit: return WholePage(clean) - stored_ref = store_full_text(page.url, clean, options.store_policy) head_budget = floor(limit * options.head_ratio) tail_budget = limit - head_budget @@ -132,10 +131,16 @@ reduce_web_extract(page, options): tail = clean[len(clean)-tail_budget:] head = snap_head_to_previous_line_boundary(head) tail = snap_tail_to_next_line_boundary(tail) + body = head + omission_marker + tail + + put = ccr_store.put(clean) + offload = OffloadOutput::from_retained_put(body, Html, put) + if offload is unavailable: + return WholePage(clean) return TruncatedPage { - text: head + omission_marker + tail + retrieval_footer(stored_ref), - ref: stored_ref, + text: offload.body + retrieval_footer(offload.token), + token: offload.token, original_chars: len(clean), inline_chars: len(text), } @@ -168,35 +173,43 @@ can decide whether to inspect them with a vision or browser tool. ## Storage -TinyJuice should reuse the artifact-store design from -`tokenjuice-improvement-spec.md` rather than writing arbitrary filesystem paths -from core compressor code. +TinyJuice uses CCR for model-facing recovery of omitted web text. The full +cleaned page is retained through the same `CcrStore` trait used by other lossy +reducers, and the footer is emitted only after `OffloadOutput::from_retained_put` +proves the store retained the original. + +OpenHuman operator-facing artifacts remain a host concern. Core TinyJuice does +not write arbitrary filesystem paths from web reducer code and does not fetch +URLs. -Stored full-text records should include: +CCR metadata exposed by `WebExtractReduction` includes: ```text -WebExtractArtifact { - id: ArtifactId, +WebExtractReduction { + ccr_token: Option, source_url_hash: String, - source_host: String, - content_sha256: String, - content_bytes: usize, - content_chars: usize, + source_host: Option, + original_chars: usize, + inline_chars: usize, + head_chars: usize, + tail_chars: usize, + omitted_chars: usize, format: WebExtractFormat, - created_at: Timestamp, - expires_at: Option, + truncated: bool, + full_text_retained: bool, + base64_images_replaced: usize, } ``` Storage requirements: -- Artifact ids must be generated by TinyJuice, not derived directly from URLs. +- CCR tokens must be generated by TinyJuice, not derived directly from URLs. - URL hostnames may appear in display metadata, but paths and filenames should be hash-based or strictly sanitized. -- Disk storage must use private file permissions where the platform supports - them. -- A failed store must degrade to a truncated result with a clear footer saying - that full recovery is unavailable. +- Disk-backed CCR storage must use private file permissions where the platform + supports them. +- A failed CCR store must degrade to the cleaned whole page with no footer, + never to an unrecoverable truncated page. - Disk-backed retention should have size and TTL controls. ## Retrieval Footer @@ -208,15 +221,13 @@ Example: ```text -------- [TRUNCATED] -------- Showing 11,250 chars (head) + 3,750 chars (tail) of 85,420 total clean characters. -Full text artifact: tj-web-01H... -To read the omitted middle: tinyjuice_retrieve id="tj-web-01H..." offset= limit= +Full text token: (marker ⟦tj:⟧). +To read the omitted middle: tokenjuice_retrieve token="" offset= limit= ----------------------------- ``` -If the host exposes file-based retrieval instead of an artifact API, the adapter -may render a host-native call such as `read_file path="..." offset= -limit=`. TinyJuice core should prefer opaque artifact ids so it does not leak -local cache paths into model context by default. +TinyJuice core uses opaque CCR tokens so local cache paths are not leaked into +model context. Footer requirements: diff --git a/plan/content-compressor-roadmap.md b/plan/content-compressor-roadmap.md index 87c5e6f..081b261 100644 --- a/plan/content-compressor-roadmap.md +++ b/plan/content-compressor-roadmap.md @@ -14,17 +14,40 @@ Current behavior: - Keeps all rows for small arrays. - For large arrays, keeps fixed head/tail windows plus error rows and numeric outliers. +- Heterogeneous object arrays can render as smaller per-shape bucket tables, + preserving original row indices and using CCR recovery when bucket rows are + dropped. +- Full-present constant columns are hoisted into table metadata instead of + repeated in every rendered row. +- Analyzer reports field frequencies, types, unique counts/ratios, constants, + sparse fields, numeric stats, estimated table bytes, and rough estimated + reduction. +- Planner preserves dynamic head/tail anchors, query-relevant matches, + structural sparse rows, numeric outliers/change points, discriminator values, + duplicate and near-duplicate representatives, information-dense rows, and + deterministic spread anchors. +- Adaptive spread anchor counts use deterministic semantic-bigram saturation. +- Nested uniform objects flatten into dotted columns. +- Stringified JSON objects are parsed and flattened when they are small and + valid. +- `SmartCrusherTableTransform` exposes faithful table rendering as a typed + reformat that runs without CCR. +- `SmartCrusherRowsTransform` exposes row-dropping SmartCrusher output as a + typed offload that emits output only with a retained CCR token. Add: -- Analyzer for key frequencies, field types, constants, unique ratios, numeric - stats, sparse fields, and estimated reduction. -- Planner for dynamic anchors, query matches, structural outliers, discriminator - buckets, and duplicate clusters. -- Adaptive keep count using deterministic saturation/knee detection. -- Heterogeneous-array handling through buckets. -- Nested uniform object flattening into dotted columns. -- Stringified JSON parsing where safe. +- Broader fixture coverage for all SmartCrusher planner paths. + +Status: initial router-level fixture coverage now includes a query-anchored +row-dropping case with CCR recovery. Sparse structural rows, error rows, +numeric outliers, and duplicate-cluster representative anchors now also have +router-level fixture coverage. Constant-column hoisting, nested object +flattening, stringified JSON flattening, heterogeneous bucket rendering, +discriminator anchors, numeric change-point anchors, and information-dense +anchors now also have router-level fixture coverage. Latest/oldest query +direction, exact and near-duplicate clusters, deterministic spread anchors, and +adaptive spread saturation now cover the remaining current planner anchor paths. Do not add: @@ -45,14 +68,20 @@ Current behavior: - Keeps structural lines and changed lines. - Collapses long unchanged context. - Summarizes noisy lockfile/bundle hunks inside `DiffCompressor`. +- Exposes `DiffNoiseTransform` as a typed offload transform that emits lossy + diff views only with a retained CCR token. +- Supports configurable noisy path substrings, optional whitespace-only hunk + dropping, per-hunk reason markers, and droppable-body bloat estimates. Add: -- Separate `DiffNoise` offload transform. -- Configurable suffix list for lockfiles and generated bundles. -- Optional whitespace-only hunk dropping. -- Per-hunk omission reasons. -- Bloat estimate based on droppable body-byte share. +- Broader fixture coverage for host-tuned noisy path policy. + +Status: router-level fixture coverage now includes lockfile and generated-bundle +noise cases that assert reason markers and recovery tokens. Host-tuned policy +fixtures now cover configured noisy path patterns, whitespace-only hunk +dropping, semantic-hunk preservation, and CCR retention through the typed +DiffNoise transform. Do not add: @@ -73,14 +102,23 @@ Current behavior: - Command output uses the rule reducer. - Non-command logs use signal preservation for errors, warnings, summaries, and stack traces. +- Repetitive non-command logs use a lossless, reconstructible template reformat + before signal offload. +- `LogTemplateTransform` exposes the template path as a typed reformat that + runs without CCR. +- `SignalLogTransform` exposes signal-preserving log compression as a typed + offload that emits output only with a retained CCR token. Add: -- Lossless log-template reformat before signal offload. -- Reconstructible template blocks and variants. - More command-rule parity for common build systems and CI outputs. - GitHub Actions failing-step summaries through rule metadata. +Status: partially implemented. `ci/github-actions` now handles +`gh run view --log` output, retaining failing step/error lines and counting +failed-step annotations through the fixture suite. Log-template fixture +coverage now reconstructs original lines from emitted template blocks. + Do not add: - Signal compression for data that has no log signal. @@ -99,14 +137,15 @@ Current behavior: - Tree-sitter path for Rust, TypeScript/JavaScript, and Python behind feature. - Brace-depth heuristic fallback. - Collapses function/method bodies. +- Explicit stub modes cover signatures-only, public API, matched symbols, and + expand-around-lines. +- Stub output carries elision metadata with line ranges and parse status. +- Host read intent distinguishes exact reads from explicit stub reads. +- `CodeStubTransform` exposes explicit source-code stubbing as a typed offload + that emits output only with a retained CCR token. Add: -- Explicit stub modes: signatures-only, public API, matched symbols, and - expand-around-lines. -- Elision metadata with line ranges. -- Parse status in reports. -- Host intent that distinguishes exact read from stub read. - Additional language parsers only behind features. Do not add: @@ -127,15 +166,24 @@ Current behavior: - Parses grep/ripgrep-style `path:line:body`. - Groups by file. - Keeps top K matches per file. -- Scores query-term density or line salience. +- Scores query relevance through the shared BM25 scorer, falling back to line + salience without a query. +- `SearchTransform` exposes search-result thinning as a typed offload that + emits output only with a retained CCR token. +- Ranked search-read helpers score exact symbols, paths, match density, + imports/exports, generated/vendor penalties, omitted counts, and merged + snippet windows over host-provided matches. Add: -- Shared BM25 scorer. - Better query context propagation. -- Path and generated/vendor penalties for search-read adapter. -- Omitted match counts by file and global total. -- Snippet window merging for host-side search-read. +- Broader fixture coverage for search-read host adapters. + +Status: initial router-level fixture coverage now includes a query-ranked +search thinning case with omitted counts and CCR recovery. Ranked search-read +fixtures now cover exact-symbol/path/density ordering, vendor/generated +penalties, bounded snippet-window merging, and explicit omitted-match counts for +host-provided candidates. Do not add: @@ -154,12 +202,20 @@ Current behavior: - Single-pass HTML-to-text extractor. - Drops scripts, styles, head, noscript, and SVG. +- `HtmlExtractTransform` exposes lossy HTML-to-text extraction as a typed + offload that emits output only with a retained CCR token. +- Web-extract reducer accepts already extracted text/markdown/HTML, strips + inline base64 image payloads, preserves metadata, and keeps recovery footers + attached to truncated pages. Add: - More complete entity support if fixtures show need. -- Web-extract reducer as an adapter-facing module for already extracted text. -- Base64 image stripping for web extraction outputs. +- Broader fixture coverage for web-extract host formats. + +Status: initial web-extract fixture coverage now includes a markdown host +format case for base64 image replacement, real image URL preservation, URL +secret redaction, and retained recovery text. Do not add: @@ -179,14 +235,23 @@ Current behavior: - Optional ML text compressor through host callback. - ML disabled by default. - Generic fallback only runs for command output. +- Deterministic extractive TextCrusher keeps verbatim spans scored by recency, + BM25 query relevance, identifiers, numbers, and error/salience markers. +- Near-duplicate suppression uses word-term overlap. +- `TextCrusherTransform` exposes the deterministic path as a typed offload + that emits output only with a retained CCR token. +- Custom tag protection wraps ML compression so protected workflow tags must + survive byte-for-byte. Add: -- Deterministic extractive TextCrusher. -- Segment scoring by recency, BM25 query relevance, identifiers, numbers, and - error/salience markers. -- Near-duplicate suppression with word shingles. -- Tag protection around ML compression before enabling broader use. +- Broader fixture coverage for TextCrusher scoring and ML tag protection. + +Status: initial router-level fixture coverage now includes a query-relevant +error-preservation case with verbatim signal text and CCR recovery. ML +tag-protection fixtures now prove custom workflow tags are restored byte-for-byte +when callbacks keep protected placeholders and that ML output is declined when a +callback drops them. Do not add: @@ -198,4 +263,3 @@ Acceptance: - Output consists only of verbatim spans. - Custom workflow tags survive ML compression byte-for-byte. - Plain data without useful signal declines rather than blindly truncating. - diff --git a/plan/conversation-compression-plan.md b/plan/conversation-compression-plan.md index 50d2414..f324260 100644 --- a/plan/conversation-compression-plan.md +++ b/plan/conversation-compression-plan.md @@ -65,6 +65,14 @@ Add pure helpers first: - latest visible assistant anchor - manual partial split and rejoin helpers +Status: implemented in core. The helpers cover provider-safe JSON argument +shrinking, deterministic tail selection, protected head decay, +tool-call/tool-result boundary alignment, orphan sanitizer, latest real +user/visible assistant anchors, and partial split/rejoin with orphan +sanitization after middle replacement. Last-N real user exchange helpers now +select a deterministic retained tail while skipping hidden/internal summary +messages. + Acceptance: - JSON argument shrinking preserves parseability. @@ -107,6 +115,12 @@ Acceptance: - Cache hints are reported separately from compression steps. - Frozen byte ranges are left for adapters to splice byte-identically. +Status: implemented in core. `cache_hints.rs` exposes stable prefix keys and +provider marker hints without mutating payloads. `live_zone.rs` adds +provider-neutral frozen-prefix/mutable-block byte ranges, validates replacement +splicing without rewriting frozen bytes, and reports UUID/timestamp/JWT/hash +volatility using redacted findings only. + ## P2: Summary Contract Add optional summary interfaces: @@ -159,4 +173,3 @@ Acceptance: - Session database writes. - Compression leases implemented against OpenHuman storage inside core. - Prompt cache mutation mixed into lossy compression APIs. - diff --git a/plan/current-state-and-critique.md b/plan/current-state-and-critique.md index 1df9bcb..8b2c546 100644 --- a/plan/current-state-and-critique.md +++ b/plan/current-state-and-critique.md @@ -43,10 +43,9 @@ be added as rules or rule-engine features before adding one-off compressor code. ### README Drift -`README.md` still says the repository is intentionally blank scaffolding and -that real strategies have not landed. That is no longer true. The docs should -be updated after implementation plans turn into code, but this plan does not -edit README because the user requested plan files only. +`README.md` now reflects that deterministic reducers, content-aware compressors, +and CCR recovery plumbing exist. Keep future public API contract changes +documented there or under `docs/` as implementation plans turn into code. ### Duplicate Compressor Concepts @@ -65,20 +64,33 @@ compress content when an adapter passes an extension hint such as `rs`, `py`, or normal exact reads. OpenHuman should provide a host policy that marks exact file reads as raw unless the user or tool intentionally requests a stub/summary. -### Lossy Safety Is Runtime-Checked, Not Type-Checked +### Lossy Safety Is Partly Type-Checked -`CompressOutput::lossy` is currently a flag, and `route()` is responsible for -checking CCR availability. This works, but it is too easy for future code to -accidentally bypass `route()` or return lossy text without recoverability. The -Headroom-style typed pipeline should make offload-backed transforms structurally -different from lossless reformats. +`CompressOutput::lossy` is still the compatibility flag for existing +compressors, and `route()` remains responsible for checking CCR availability. +The new `pipeline` module adds the first typed split: +`ReformatTransform` emits ordinary `TransformOutput`, while `OffloadOutput` +can only be built from a retained `CcrPutResult`. The production router now +uses those typed transforms for covered non-command JSON, diff, log, search, +HTML, explicit code-stub, and TextCrusher paths. Command reducers, ML text, +non-stub code compression, and generic fallback behavior still use the +compatibility compressor interface. -### Global CCR Store Makes Tests And Policy Harder +### CCR Store Is Now Injectable -The global store is convenient for integration, but it makes isolated tests, -adapter-specific policies, and future backends harder. An injectable `CcrStore` -trait should be introduced while preserving the current global functions as -compatibility wrappers. +The process-global store remains for compatibility, but `CcrStore`, +`GlobalCcrStore`, and `MemoryCcrStore` now exist. `compress_content_with_store` +and `route_with_store` let tests and host adapters exercise CCR behavior +against an isolated store instead of the global cache. Remaining work is to +thread explicit stores through more OpenHuman call sites instead of relying on +the global compatibility path. + +### Pipeline Reports Cover Typed And Compatibility Paths + +`compress_content_with_store_report` and `route_with_store_report` now return a +redacted `PipelineReport` with byte counts, cheap bloat estimate, applied step, +CCR token ids, lossy flag, and skip reason. Typed production paths report their +transform names, while remaining compatibility paths report `compat_router`. ### Missing Machine Protocol @@ -86,51 +98,45 @@ The Rust crate has compatible types, but no stable `reduce-json` protocol or CLI. That blocks non-Rust hosts and makes OpenHuman integration harder to test from fixtures. -### Missing Safe-Inventory Policy +### Safe-Inventory Policy Implemented -The code has a narrow `is_file_content_inspection_command()` guard. It does not -yet implement the richer policy from the references: safe inventory pipelines -may compact, exact content reads stay raw, mixed shell sequences stay raw, and -unsafe actions such as `find -exec` stay raw. +The code now has a host-selectable `ShellCompactionPolicy` with the richer +reference behavior: safe inventory pipelines may compact, exact content reads +stay raw, mixed shell sequences stay raw, and unsafe actions such as +`find -exec` stay raw. The legacy `is_file_content_inspection_command()` helper +remains as a compatibility wrapper for older call sites. -### Footer Placement Is A Fragile Host Contract +### Footer Placement Requires A Host Contract -`route()` appends the recovery footer to the end of the compacted text and -returns a single string. Hosts that apply their own truncation caps after -compaction (OpenHuman applies a per-tool char cap and a 16 KiB byte-cap -backstop after the TokenJuice stage) keep the head and cut the tail, which -severs the footer and leaves an unrecoverable lossy view. `CompressedOutput` -should expose the footer (or the body/footer split) as separate fields so -hosts can truncate the body and reattach the marker. Until then, every host -integration must be audited for post-compaction truncation. +`CompressedOutput` now exposes both the compatibility `text` field and separate +`body`/`recovery_footer` fields. OpenHuman's tinyagents middleware composes +those fields by truncating only the body and reattaching the footer after its +per-tool char cap and byte-cap backstop. Other host integrations still need to +follow the same contract whenever they apply post-compaction caps. -### The Rule Catalog Is Inert Without Arguments +### The Rule Catalog Depends On Arguments -`compact_output_with_policy` forwards `arguments: None, exit_code: None`, and -the rule reducer only fires for command output. OpenHuman currently calls this -minimal wrapper, so the 100-rule catalog, extension hints, query hints, and -exit-code handling are all dead at the only production call site. The plan's -Phase 1 must treat migrating that call site as the primary deliverable, not a -confirmation task. +`compact_output_with_policy` still forwards `arguments: None, exit_code: None` +for minimal call sites, but OpenHuman's tinyagents middleware now captures tool +arguments in `before_tool` and calls the full policy adapter after the tool +returns. That activates command rules, extension/query hints, and exit-code +handling on the production tinyagents path. Future host adapters should avoid +the minimal wrapper unless they genuinely lack tool metadata. ### Limited Observability -Current stats expose bytes and compressor labels, and savings callbacks estimate -tokens. Missing pieces include skip reasons, applied-step traces, CCR retention -status, omission counts, and fixture-backed benchmark reports. +Current stats expose bytes, compressor labels, pipeline skip reasons, +body/footer status, CCR token ids, and class-labeled savings records. Remaining +observability work is mostly product-facing: fixture-backed benchmark reports, +stable CLI/RPC output, and richer per-compressor omission metadata in every +host surface. ## What Should Be Added -- Type-level separation of reformat and offload transforms. -- Injectable CCR store trait with current memory/disk behavior preserved. - Stable `reduce-json` request/response protocol and minimal CLI. -- Safe-inventory command policy before wider shell compaction. -- OpenHuman adapter tests for `full`, `light`, `off`, recovery tool bypass, - exact file reads, and config reload. - Deterministic benchmark and fixture suite before claims. -- Query context and BM25 scoring as a lightweight shared primitive. -- Better compressor metadata: omitted rows, omitted lines, skip reasons, and - applied transforms. +- Stable product-facing reports for omitted rows, omitted lines, skip reasons, + applied transforms, and fixture-vs-live savings classes. ## What Should Not Be Added Yet @@ -143,4 +149,3 @@ status, omission counts, and fixture-backed benchmark reports. - Host installers before `reduce-json`, validation, and doctorable policy are stable. - Compression percentage marketing claims. - diff --git a/plan/openhuman-algorithm-port-plan.md b/plan/openhuman-algorithm-port-plan.md index aad81a2..48be70a 100644 --- a/plan/openhuman-algorithm-port-plan.md +++ b/plan/openhuman-algorithm-port-plan.md @@ -8,7 +8,7 @@ algorithm this plan names the TinyJuice core modules, the OpenHuman files to add or change, the wiring, and the acceptance criteria — in enough detail that an implementing agent can execute it without re-deriving the analysis. -Verified against the OpenHuman repo (`../openhuman-2`) on 2026-07-04. All +Verified against the OpenHuman repo (`../openhuman-4`) on 2026-07-06. All OpenHuman paths below exist today unless marked NEW. ## Ground Rules @@ -59,12 +59,16 @@ the process-global cache. TinyJuice files (per pipeline-and-ccr-plan): -- `src/pipeline/mod.rs`, `src/pipeline/transform.rs`, `src/pipeline/report.rs` (NEW) -- `src/cache/store.rs` — extract `CcrStore` trait, keep global wrappers +- `src/pipeline/mod.rs`, `src/pipeline/transform.rs`, `src/pipeline/report.rs` + — present with the first transform/report primitives. +- `src/cache/store.rs` — `CcrStore`, `GlobalCcrStore`, and `MemoryCcrStore` + are present; global wrappers are preserved. OpenHuman files: -- `vendor/tinyjuice` — submodule bump (currently `4b1a34f`, behind HEAD). +- Current `../openhuman-4` has a local `src/openhuman/tokenjuice/` copy rather + than `vendor/tinyjuice`; mirror store/pipeline exports there until the crate + dependency boundary is restored. - `src/openhuman/tokenjuice/mod.rs` — re-export `CcrStore`, `PipelineReport`; `install_from_config` constructs the store explicitly instead of relying on global state, keeping the global as compatibility path. @@ -78,6 +82,15 @@ Acceptance: - Lossy output is unconstructable without a verified token (compile-time). - No behavior change for existing compaction paths (fixture parity). +Status: implemented. TinyJuice has the typed pipeline/report layer and +`CcrStore` trait with global and in-memory stores; OpenHuman mirrors those +exports under `src/openhuman/tokenjuice/`, including store-injected +`compress_content_with_store*` and `route_with_store*` paths. `install_from_config` +maps runtime config into the explicit TokenJuice install path while preserving +the global store compatibility path. OpenHuman tests cover isolated in-memory +offload behavior, CCR rejection, pipeline reporting, and runtime config/cache +limit installation. + ## P0-2: Fix The Hook, Then Safe Shell Policy Source specs: `shell-output-intercept-spec.md`, @@ -123,6 +136,17 @@ Acceptance: - A shell result carrying argv reaches the rule reducer (end-to-end test in OpenHuman, not just crate-level). +Status: implemented. TinyJuice exposes `ShellCompactionPolicy` and command +classification in `src/policy/shell.rs`; OpenHuman config surfaces +`TokenjuiceShellPolicy` with the `allow_safe_inventory` default and partial +updates/env overlays. `ToolOutputMiddleware` now captures tool arguments in +`before_tool`, parses rendered shell exit status in `after_tool`, calls +`compact_tool_output_with_policy_detail`, bypasses recovery tools, applies body +caps, and reattaches the TokenJuice recovery footer after truncation. OpenHuman +middleware tests cover argument threading, nonzero exit/stderr preservation, +safe inventory compaction, exact shell/file reads, unsafe `find -exec` +passthrough, and footer survival across both per-tool and shared byte caps. + ## P0-3: Hermes Deterministic Conversation Primitives Source spec: `hermes-compression-algorithms-spec.md`. Design detail exists in @@ -177,12 +201,24 @@ Acceptance: assistant reply. - Digest output contains no raw secrets (redaction before persistence). +Status: implemented for the deterministic subset. TinyJuice exposes budget, +boundary, provider-neutral conversion, tool-result digest, JSON string-leaf +shrinking, and redaction helpers. OpenHuman uses those helpers in +`TokenjuiceMicrocompactMiddleware` to digest older tool results in one pass, +and `ContextCompressionMiddleware` uses TokenJuice tail-budget selection, +latest-user/latest-assistant anchors, and tool-boundary alignment before +summarization. Tests cover retained anchors, tool result digesting, +duplicate-result handling, parseable/redacted JSON arguments, and secret +redaction before model-facing/persisted output. + ## P1-1: Savings Accounting Upgrade Source spec: `savings-accounting-spec.md`. OpenHuman already persists savings -(`src/openhuman/tokenjuice/savings.rs`, dashboard-facing) and the README -already claims "up to 80% fewer tokens" — this port is what makes that claim -either verifiable or correctable, so it lands early in P1. +(`src/openhuman/tokenjuice/savings.rs`, dashboard-facing). The old reviewed +plan assumed OpenHuman's README still made a broad percentage-savings claim; +as of 2026-07-06 that claim is no longer present. This slice still lands early +because live estimates, measured usage, and future fixture benchmark results +must be labeled separately before any savings surface can be trusted. TinyJuice files: @@ -205,6 +241,13 @@ Acceptance: - Measured vs estimated is visible end-to-end (crate → adapter → RPC schema). - Fixture benchmark results are reported separately from live stats. +Status: implemented. TinyJuice exposes class-labeled `SavingsRecord` values +with live vs fixture-benchmark source labels, byte metadata, lossy/CCR flags, +rule/skip metadata, and optional measured model usage. OpenHuman mirrors the +records, persists rollups by model, compressor, accounting class, record +source, and content kind, exposes those labels through `tokenjuice.savings_*` +controllers, and records provider `UsageInfo` as measured metadata-only usage. + ## P1-2: Web Extract Truncate-Store Source spec: `web-extract-truncate-store-spec.md`. OpenHuman has the host @@ -220,8 +263,9 @@ TinyJuice files: (`![alt](data:image...)` → `[IMAGE: alt]`, real URLs kept); small pages returned whole; large pages: offload full text via `CcrStore`, return line-snapped head/tail (head ratio 0.75 of the char limit), omission marker - and retrieval footer. Implemented as an `OffloadTransform` so it cannot emit - a footer without a retained original. + and retrieval footer. The reducer gates footer emission through + `OffloadOutput::from_retained_put`, so it cannot emit a footer without a + retained original. - `src/types.rs` — `WebExtractReduceInput { url, title, content, format, char_limit, metadata }` and batch shape with a combined inline budget. @@ -241,6 +285,11 @@ OpenHuman files: `char_limit` (default 15000, clamp 2000..500000), `convert_base64_images` (default true). +Status: implemented. OpenHuman has the shared reducer wired through both +`web_fetch` and `http_request`, and `[tokenjuice.web_extract]` is the canonical +config shape. Legacy flat `web_extract_*` keys, env vars, and settings patches +remain accepted and are normalized before live updates. + Acceptance: - Base64 image bytes absent from output and metadata; real image URLs remain. @@ -254,8 +303,9 @@ Acceptance: Source specs: `ast-stub-read-spec.md`, roadmap detail in `plan/content-compressor-roadmap.md`. The crate already has tree-sitter code compression behind `tokenjuice-treesitter` (OpenHuman enables it by default in -`Cargo.toml`). What is missing is explicit modes and — critically — host -intent, so exact reads are never stubbed by accident. +`Cargo.toml`). Current TinyJuice/OpenHuman state has explicit stub modes and +host intent wired: exact reads remain byte-exact by default, while stub modes +must be requested explicitly. TinyJuice files: @@ -287,6 +337,15 @@ Acceptance: ranges are listed with line numbers. - Parse failure reports fallback and never silently omits without markers. +Status: implemented. TinyJuice has explicit `ReadIntent::Exact` as the default, +`ReadIntent::Stub(StubMode)`, line-range metadata, and parse-status reporting +for tree-sitter and heuristic fallback paths. OpenHuman `file_read` exposes a +`mode` argument (`full`, `stub`, `signatures`, `public_api`, `symbols`, `lines`) +and calls `stub_code` directly for requested stub modes while `full` remains +byte-exact. Tool and router tests cover schema guidance, stub output, symbol +expansion, heuristic fallback reporting, partial-read tracking, exact default +router passthrough, and full middleware-chain exactness for `file_read`. + ## P1-4: Core Compressor Upgrades (SmartCrusher, DiffNoise, Log Templates, TextCrusher, BM25) Source specs: both headroom specs. Design detail already exists in @@ -306,6 +365,15 @@ Acceptance: as specified per compressor in the roadmap plan; plus one OpenHuman end-to-end fixture per compressor family driven through `compact_tool_output_with_policy`. +Status: implemented. TinyJuice has host-hook e2e coverage through +`compact_tool_output_with_policy` for SmartCrusher, DiffNoise, search +thinning/BM25 query ranking, TextCrusher, HTML extraction, and generic command +fallback, each asserting recoverability where the path is lossy. OpenHuman +`../openhuman-4` mirrors that host-adapter coverage in +`src/openhuman/tokenjuice/tool_integration.rs`; commit `88a17f9f8` adds the +remaining HTML extraction and generic command fallback fixtures so each current +compressor family is exercised through the adapter path. + ## P2-1: Ranked Search-Read Tool Source spec: `ranked-search-read-spec.md`. This is a NEW host tool, not a @@ -323,12 +391,18 @@ TinyJuice files: OpenHuman files: -- `src/openhuman/tools/impl/filesystem/search_read.rs` (NEW) — one tool: +- `src/openhuman/tools/impl/filesystem/search_read.rs` — one tool: glob → grep → rank via TinyJuice scoring → bounded snippets with path/line metadata → omission report. Registered alongside the existing tools; the schema description positions it as the preferred first move over separate glob/grep/read calls. - `src/openhuman/tools/impl/filesystem/mod.rs` — registration. +- `src/openhuman/tools/user_filter.rs` and + `app/src/utils/toolDefinitions.ts` — include `search_read` in the file-read + tool family so user tool preferences and frontend toggles do not hide the + ranked read surface when "Read Files" is enabled. +- `app/src/utils/toolTimelineFormatting.ts` — render `search_read` as a + known search/read timeline tool. Acceptance: @@ -337,6 +411,13 @@ Acceptance: counts; vendor/generated paths deprioritized unless requested. - The tool never returns whole files; follow-up reads use `file_read`. +Status: implemented. TinyJuice has pure ranking and snippet-window fixtures for +exact symbol/path/density ordering, vendor/generated penalties, bounded window +merging, and omitted counts. OpenHuman `../openhuman-4` already carried the host +`search_read` implementation and registration; commit `76399a6ce` wires it into +the Rust user-preference family, frontend tool catalog, and timeline formatting +so the registered tool is exposed consistently with `file_read`. + ## P2-2: Subagent Summary Evidence Extraction Source spec: `subagent-summary-spec.md`. OpenHuman already has @@ -365,6 +446,18 @@ Acceptance: - Summarizer failure path returns the deterministic summary, never the full transcript, and never loses the subagent's final answer. +Status: implemented. TinyJuice exposes provider-neutral subagent transcript +events, deterministic evidence extraction, bounded markdown rendering, omission +reports, and fixtures for long transcripts, failure evidence, contradictory +findings, and byte-budget truncation. OpenHuman `../openhuman-4` constructs +`SubagentRunOutcome::deterministic_summary` from the child run history, uses the +shared handoff renderer for completed and incomplete `spawn_subagent` / +`continue_subagent` results, preserves the final answer when it is not already +inside the deterministic summary, and falls back to the deterministic +checkpoint when the LLM checkpoint summary is empty or fails. The +`PayloadSummarizer` also supplies this deterministic scaffold for subagent-like +payloads before invoking its semantic summarizer. + ## P2-3: Summary Contract And Prompt Cache Hints Source spec: `hermes-compression-algorithms-spec.md` (P2 portions). Builds on @@ -372,7 +465,9 @@ P0-3. TinyJuice adds `src/conversation/summary.rs` (SummaryProvider trait, strict structured schema, deterministic fallback, failure policy: auth/network failures abort and preserve messages) and `src/cache_hints.rs` (static-prefix cache key = SHA-256 of instructions + NUL + sorted tool -schemas; Anthropic carrier hints only, no payload mutation). OpenHuman's +schemas; Anthropic carrier hints only, no payload mutation) plus +`src/live_zone.rs` for frozen-prefix byte ranges, mutable replacement splicing, +and redacted volatile-value detection. OpenHuman's `ContextCompressionMiddleware` implements `SummaryProvider` over its existing summarization path and adopts the failure policy; cache hints feed the provider layer (`src/openhuman/inference/provider/`) as routing hints only. @@ -382,6 +477,17 @@ tag + end marker); repeated compactions update the previous summary instead of nesting; tool order does not change the cache key; frozen prefix bytes are never rewritten. +Status: implemented in core and mirrored in OpenHuman. TinyJuice exposes the +`SummaryProvider` contract, strict `StructuredSummary` shape, summary metadata +tag/end marker, deterministic fallback, summary upsert helpers, stable prompt +cache hints, and live-zone frozen-prefix splicing. OpenHuman +`../openhuman-4/src/openhuman/tokenjuice/` mirrors those modules, while +`src/openhuman/tinyagents/summarize.rs` uses TokenJuice boundary/tail planning, +the summary failure policy, deterministic trim fallback, the canonical summary +end marker, and duplicate-summary removal before rewriting model requests. +Cache hints and live-zone helpers are available through the mirrored +`tokenjuice` exports for provider/adaptor use. + --- ## Deferred (with unlock conditions) diff --git a/plan/openhuman-integration-plan.md b/plan/openhuman-integration-plan.md index 2688f50..ecc924e 100644 --- a/plan/openhuman-integration-plan.md +++ b/plan/openhuman-integration-plan.md @@ -7,10 +7,12 @@ keeping the core crate independent of OpenHuman runtime types. ## Verified OpenHuman State (2026-07-04) -The integration is not greenfield. OpenHuman already vendors TinyJuice -(`vendor/tinyjuice` submodule pinned at `4b1a34f`, wired through -`[patch.crates-io]`) and ships it as the "TokenJuice" product feature. The -existing surface: +The integration is not greenfield. The current `../openhuman-4` checkout carries +a local TokenJuice mirror under `src/openhuman/tokenjuice/` and ships it as the +"TokenJuice" product feature. Earlier OpenHuman checkouts used a +`vendor/tinyjuice` submodule; in this checkout, crate contract changes are +mirrored into the local TokenJuice copy until the dependency boundary is +restored. The existing surface: - Adapter module: `src/openhuman/tokenjuice/` re-exports the crate API and owns `install_from_config(&Config)`, called at startup and on live settings @@ -20,47 +22,47 @@ existing surface: plus per-agent `AgentTokenjuiceCompression` overrides and `tokenjuice/config_patch.rs` partial updates. - Hook site: `ToolOutputMiddleware::after_tool` in - `src/openhuman/tinyagents/middleware.rs` (~line 787). Per tool result the - chain is: (1) `PayloadSummarizer` semantic summarization (orchestrator only), - (2) TokenJuice compaction, (3) per-tool char cap, (4) shared 16 KiB byte-cap - backstop with artifact persistence (`ToolResultArtifactStore`). + `src/openhuman/tinyagents/middleware.rs` (~line 787). Per normal tool result + the chain is: (1) `PayloadSummarizer` semantic summarization (orchestrator + only), (2) TokenJuice compaction, (3) per-tool char cap, (4) shared 16 KiB + byte-cap backstop with artifact persistence (`ToolResultArtifactStore`). `HandoffMiddleware` is registered after but runs first (the harness runs `after_tool` in reverse registration order) and can stash raw >50k-token - payloads before any of this. + payloads before any of this. Recovery tool results bypass this chain and are + returned exactly. - Recovery tools: both `retrieve_tool_output` - (`src/openhuman/tools/impl/system/retrieve_tool_output.rs`, legacy) and - `tokenjuice_retrieve` (`src/openhuman/tokenjuice/tools.rs`) are registered. + (`src/openhuman/tools/impl/system/retrieve_tool_output.rs`, legacy alias) and + `tokenjuice_retrieve` (`src/openhuman/tokenjuice/tools.rs`) are registered; + the legacy tool delegates to the canonical implementation. - `Auto` profile resolution is host-side in `agent/harness/definition.rs:487`: coding models resolve to `Light`, - everything else to `Full`. The crate's own `Auto == Full` mapping in - `options_for_agent()` is a silent fallback if a host forgets to resolve. - -### Known Integration Bugs (found in review) - -1. **The call site uses the minimal hook.** `middleware.rs:787` calls - `compact_output_with_policy(content, tool_name, enabled, profile)`, which - forwards `arguments: None, exit_code: None`. Consequences: command/argv - never reach the rule reducer, so the 100-rule command catalog is largely - inert in OpenHuman; extension and query hints never fire, so code/JSON/HTML - routing from file-bearing tools is disabled; exit-code-aware - failure-preserving log behavior is off. Migrating this one call site to - `compact_tool_output_with_policy(tool_name, arguments, output, exit_code, - profile)` is the single highest-value integration change. -2. **Downstream truncation can destroy recovery footers.** TinyJuice appends - its CCR footer at the *end* of compacted output. Steps 3 and 4 of the - middleware chain (per-tool char cap, byte-cap backstop) run *after* - TokenJuice and keep the head, so an output that is compacted but still over - the cap loses its footer: the model gets a lossy view with no recovery - marker while the CCR entry sits unreachable. Fix options: make the host - caps footer-aware (truncate the body, reattach the trailing marker), or - expose the footer as a separate field on `CompressedOutput` so hosts can - compose it after their own truncation. The second is the cleaner contract - and belongs in core. -3. **Vendored crate drift.** The submodule at `4b1a34f` is behind this repo's - HEAD. Any plan phase that changes footer text, marker constants, or the - hook signature must include a submodule bump plus a compatibility pass over - OpenHuman's marker parsing and tool docs (`retrieve_tool_output.rs` still - documents the legacy `retrieve_tool_output("")` sentinel). + everything else to `Full`. TinyJuice now treats unresolved `Auto` as + passthrough with `none/agent-profile-auto-unresolved`, so hosts must resolve + it before calling the adapter. + +### Resolved Integration Findings + +1. **Hook migration status.** OpenHuman's tinyagents middleware now captures + `TaToolCall` arguments in `before_tool`, parses rendered shell exit-code + prefixes, and calls the full TokenJuice policy adapter after each tool + result. Command rules, extension/query hints, and exit-code-aware log + behavior are active on that production path. Keep the minimal wrapper only + for call sites that truly lack metadata. +2. **Footer-truncation status.** `CompressedOutput` now exposes separate + `body` and `recovery_footer` fields while keeping `text` as the + compatibility body+footer output. OpenHuman's tinyagents middleware caps the + body and reattaches the footer afterward; regression tests cover both the + per-tool char cap and the shared byte-budget path. Any future host cap that + runs after TokenJuice must preserve the same body/footer contract. +3. **Recovery-tool passthrough status.** OpenHuman's tinyagents middleware now + bypasses summarization, TokenJuice compaction, per-tool caps, and the shared + byte cap for both `tokenjuice_retrieve` and legacy `retrieve_tool_output`. + A host-path regression test covers both names with intentionally tiny caps. +4. **OpenHuman checkout shape.** The reviewed plan assumed a + `vendor/tinyjuice` submodule, but this `../openhuman-4` checkout carries a + local TokenJuice copy under `src/openhuman/tokenjuice/`. Footer and hook + contract changes therefore need to be mirrored there until the dependency + boundary is restored. ## Current Fit @@ -82,7 +84,7 @@ OpenHuman should own: - calling `install_config()` at startup and config reload - passing tool arguments and exit codes into TinyJuice - exposing `tokenjuice_retrieve(token, range?)` -- ensuring recovery tool output is never compacted +- ensuring recovery tool output is never compacted or host-capped - recording stats without raw content - deciding exact-read/stub-read policy - wiring optional ML compression through the existing callback @@ -105,27 +107,24 @@ TinyJuice core should own: Tasks: -- Migrate `ToolOutputMiddleware::after_tool` from `compact_output_with_policy` - to `compact_tool_output_with_policy`, passing the tool's JSON arguments and - exit code. This activates the rule catalog, extension/query hints, and - failure-preserving log behavior that are currently dead in OpenHuman. -- Bump the `vendor/tinyjuice` submodule and reconcile marker/tool-name - constants with OpenHuman's tool docs. -- Fix the footer-truncation ordering bug: either make the per-tool char cap - and byte-cap backstop footer-aware, or (preferred) split the recovery footer - into its own `CompressedOutput` field so the host truncates the body and - reattaches the footer. +- Keep `ToolOutputMiddleware::after_tool` on the full policy adapter, passing + captured tool JSON arguments and parsed exit code. +- Reconcile marker/tool-name constants with OpenHuman's tool docs. In the + current `openhuman-4` checkout, mirror crate contract changes in the local + `src/openhuman/tokenjuice/` copy rather than a `vendor/tinyjuice` submodule. +- Preserve the footer-aware cap contract: the host truncates + `CompressedOutput::body` and reattaches `CompressedOutput::recovery_footer`. - Verify credential scrubbing runs before `after_tool` compaction; if not, compaction can retain secrets in CCR (see Risks). - Document the middleware ordering invariant: handoff observes raw output first, then summarizer, then TokenJuice, then caps. TokenJuice must tolerate receiving summarizer output rather than raw tool output. - Add OpenHuman-side tests for `full`, `light`, `off`, and `auto` resolution - (`auto` resolves in `definition.rs`, not in the crate — test the host - mapping, and consider removing the crate's silent `Auto == Full` fallback in - favor of an explicit resolution requirement). -- Add tests proving recovery-tool output bypasses compaction for both - registered tool names. + (`auto` resolves in `definition.rs`, not in the crate). TinyJuice and the + mirrored OpenHuman copy now enforce unresolved `Auto` as an explicit + passthrough, covered by `auto_agent_profile_requires_host_resolution`. +- Add tests proving recovery-tool output bypasses compaction and host caps for + both registered tool names. - Add tests proving config reload updates cache and compressor settings. Acceptance: @@ -163,11 +162,11 @@ Acceptance: Tasks: - Expose `tokenjuice_retrieve` in every agent profile that may see a footer. -- Consolidate the duplicate recovery tools: OpenHuman registers both the - legacy `retrieve_tool_output` and `tokenjuice_retrieve` with separate - implementations. Keep one implementation, alias the legacy name during - migration, and make sure footer text, tool schema docs, and - `RECOVERY_TOOL_NAMES` agree on the canonical name. +- Continue recovery-tool migration cleanup: OpenHuman registers both the + legacy `retrieve_tool_output` and canonical `tokenjuice_retrieve`, with the + legacy name delegated to the canonical implementation. Keep footer text, tool + schema docs, visibility tests, and `RECOVERY_TOOL_NAMES` aligned on + `tokenjuice_retrieve` as canonical. - Support full and ranged retrieval by lines or bytes. - Return clear not-found output for expired or evicted CCR entries. - Consider UI affordances for "retrieve full original" without requiring the @@ -179,6 +178,15 @@ Acceptance: - Ranged retrieval handles UTF-8 safely. - Not-found cases do not panic or leak local paths. +Status: behavior implemented; compatibility cleanup remains intentionally +non-breaking. New recovery footers point at canonical `tokenjuice_retrieve`, +while the legacy `retrieve_tool_output` tool stays registered as an alias for +older prompts and allowlists. Both names route through the same cache retrieval +implementation, both are included in `RECOVERY_TOOL_NAMES`, and middleware tests +cover bypassing TokenJuice compaction plus host caps for both names. Tool tests +cover full retrieval, ranged retrieval, and not-found errors without local path +leaks. + ### Phase 4: Stats And Cost Integration Tasks: @@ -234,11 +242,11 @@ Acceptance: off by default until tag protection and fixtures exist. Note the Kompress bridge is already wired in OpenHuman's `install_from_config`; verify its config default is off. -- OpenHuman's README already claims "up to 80% fewer tokens" for TokenJuice, - which conflicts with this plan set's no-percentage-claims-until-fixtures - constraint. Either land the fixture benchmark suite early enough to back the - claim or soften the README wording. -- The host truncation caps (per-tool char cap, 16 KiB byte-cap backstop) and - TinyJuice compaction currently compose blindly. Until the footer contract is - fixed, any cap below the compacted size silently severs recoverability. - +- Older plan notes assumed OpenHuman's README claimed a broad TokenJuice + percentage saving. Rechecked on 2026-07-06: that claim is no longer present. + Keep it that way until fixture benchmark reports exist, and keep those + reports separate from live savings stats. +- Host truncation caps must preserve the TinyJuice body/footer contract. The + current tinyagents per-tool char cap and 16 KiB byte-cap backstop truncate + only the compacted body and reattach the recovery footer; future host cap + paths need the same invariant. diff --git a/plan/pipeline-and-ccr-plan.md b/plan/pipeline-and-ccr-plan.md index ecd263c..f65522b 100644 --- a/plan/pipeline-and-ccr-plan.md +++ b/plan/pipeline-and-ccr-plan.md @@ -88,6 +88,11 @@ Skip reasons must not include raw content. - Move token validation into reusable store helpers. - Add isolated tests using an in-memory store. +Status: implemented in the core crate. `CcrStore`, `GlobalCcrStore`, +`MemoryCcrStore`, and compatibility wrappers exist; isolated memory-store tests +cover global isolation, oversized rejection, range retrieval, and malformed +token rejection. + ### Step 2: Add Pipeline Compatibility Wrapper - Build a pipeline path that can run current compressors as compatibility @@ -95,6 +100,17 @@ Skip reasons must not include raw content. - Keep `compress_content()` and `route()` signatures intact. - Assert current behavior is unchanged through existing fixtures. +Status: partially implemented. `compress_content_with_store()` and +`route_with_store()` are store-injected compatibility paths while the old APIs +delegate to `GlobalCcrStore`. `_report` variants return `PipelineReport` with +redacted skip reasons, cheap bloat estimates, and applied-step metadata. +`run_typed_pipeline()` now gives new `ReformatTransform` and `OffloadTransform` +implementations a typed execution path with no-op, reformat-only, offload-only, +mixed, and CCR-retention-failure coverage. The production router now uses the +typed path for covered non-command transforms and still falls back to the +compatibility router for command reducers, ML text, non-stub code compression, +and generic command fallback behavior. + ### Step 3: Convert Existing Compressors Gradually - JSON small-array table rendering becomes a reformat when no rows are omitted. @@ -106,6 +122,14 @@ Skip reasons must not include raw content. - Search thinning is an offload. - Code stubbing is an offload unless the caller explicitly accepts a stub view. +Status: implemented for current compressor families. JSON faithful table +rendering is exposed as `SmartCrusherTableTransform`; JSON row dropping is +exposed as `SmartCrusherRowsTransform` and requires a retained CCR token. +DiffNoise, HTML extraction, log templates, signal-log compression, search +thinning, and TextCrusher also have typed transforms. Code stubbing is exposed +as `CodeStubTransform` and requires explicit construction with a `StubMode`. +The router reports typed transform step names for these production paths. + ### Step 4: Add Bloat Estimation Add cheap estimators for: @@ -120,6 +144,13 @@ Add cheap estimators for: The router can use estimators to decide whether to try transforms and to report why a transform was skipped. +Status: first pass implemented. `pipeline::estimate_bloat` reports a redacted +0-100 score plus categorical reason for JSON rows, diff context, log +repetition, search fanout, HTML markup, code body bulk, and plain-text +repetition. The router reports the estimate and uses a zero-score estimate to +skip low-signal payloads when there is no command context, query hint, or +explicit stub-read intent. + ## Acceptance Criteria - Existing public APIs continue to compile. @@ -138,4 +169,3 @@ why a transform was skipped. complete. - Do not log raw content in pipeline reports. - Do not make all compressors implement the new traits in one large PR. - diff --git a/plan/reference-algorithm-summary.md b/plan/reference-algorithm-summary.md index eae5e1a..cf38fd8 100644 --- a/plan/reference-algorithm-summary.md +++ b/plan/reference-algorithm-summary.md @@ -14,8 +14,8 @@ TokenJuice contributes the deterministic command-output reducer model: TinyJuice already ports the reducer core and much of the rule model. The most important remaining TokenJuice work is parity hardening: missing rules, fixture -verification, `reduce-json`, safe shell inventory policy, artifact metadata, and -CLI product surface. +verification, `reduce-json`, artifact metadata, and CLI product surface. Safe +shell inventory policy is now implemented in the Rust path. ## Headroom @@ -31,8 +31,10 @@ Headroom contributes the strongest architectural contracts: - protect custom workflow tags before ML compression TinyJuice already has Headroom-inspired compressors for JSON, code, logs, diffs, -search, and HTML. The next useful ingestion is the pipeline contract and policy -model, not a wholesale port. +search, HTML, TextCrusher/BM25, pipeline reports, injectable CCR stores, and +the reformat/offload transform split. Remaining Headroom work is hardening with +fixtures and exposing the report model through stable product surfaces, not a +wholesale port. ## Hermes @@ -48,6 +50,7 @@ TinyJuice's content router. Its useful algorithms are: - structured handoff summaries - summary failure policy - prompt-cache hints and stable static-prefix cache keys +- live-zone byte-range contracts for frozen prefixes and mutable blocks - context usage breakdowns This should enter TinyJuice as an optional conversation adapter layer. It should @@ -89,9 +92,9 @@ This spec reduces full-file reads by returning source structure: - optional expansion around requested symbols or line ranges - parser fallback reporting -TinyJuice already has tree-sitter-backed code compression behind a feature. The -missing pieces are explicit modes, elision metadata, line numbers, and host -policy to avoid stubbing exact reads unintentionally. +TinyJuice has tree-sitter-backed code compression behind a feature plus explicit +stub modes, elision metadata, line numbers, parse-status reporting, and +OpenHuman host policy that keeps exact reads byte-exact by default. ## Batched Edit And Validation @@ -137,8 +140,9 @@ Savings accounting is infrastructure, not compression: - track bytes, rough tokens, measured usage, costs, elapsed time, and calls - avoid raw content in records -TinyJuice has a savings callback, but it should grow into an auditable report -model before UI claims are made. +TinyJuice has class-labeled savings records for live and fixture-benchmark +sources. UI claims should still wait for fixture benchmark reports and measured +provider usage. ## TurboQuant Vector Compression @@ -150,4 +154,3 @@ TurboQuant-style vector compression applies to embedding/index storage: This is not prompt compression. It should be deferred or isolated behind a feature/module for memory and recall indexes. - diff --git a/plan/rule-cli-and-safety-parity-plan.md b/plan/rule-cli-and-safety-parity-plan.md index 9652487..54c5c2f 100644 --- a/plan/rule-cli-and-safety-parity-plan.md +++ b/plan/rule-cli-and-safety-parity-plan.md @@ -10,7 +10,7 @@ points, and safe shell policy. Current state: -- TinyJuice vendors 100 built-in rules. +- TinyJuice vendors 101 built-in rules. - The reference spec says upstream TokenJuice has 130 non-fixture rules. - TinyJuice also has Rust/OpenHuman-specific rules. @@ -67,6 +67,12 @@ Acceptance: - `find . -exec cat {} \;` stays raw. - Mixed shell sequences stay raw unless explicitly allowed. +Status: implemented in the reducer and router policy paths. The default +`allow-safe-inventory` policy keeps exact file-content reads raw, rejects mixed +shell sequences and unsafe inventory actions, and allows safe inventory +pipelines such as `find . -type f | sort | head`. The rule fixture suite now +covers that safe-inventory path through the reducer. + ## P0: Reduce-Json Protocol Plan: @@ -85,6 +91,18 @@ Acceptance: mode, max-inline clamping, and invalid classifier. - Protocol docs exist before OpenHuman or non-Rust hosts depend on it. +Status: partially implemented. The library now exposes `ReduceJsonRequest`, +`ReduceJsonEnvelope`, `ReduceJsonResponse`, `ReduceJsonError`, +`reduce_json_str()`, and `reduce_json_request()`. The protocol accepts direct +and envelope payloads, supports the compatibility option fields, rejects +malformed JSON and NUL bytes with structured errors, and returns stable +serde-compatible response fields with optional metadata-only trace. When +`options.store` is set and CCR retains omitted raw output, the response includes +a metadata-only CCR token reference. When `options.recordStats` is set, the +protocol emits a metadata-only `SavingsRecord` through the configured recorder. +The current library contract is documented in `docs/reduce-json-protocol.md`. +Durable artifact refs remain to be implemented. + ## P1: CLI Surface Minimal order: @@ -107,6 +125,27 @@ Implementation guidance: - Pass through raw content on reducer failure unless the CLI command explicitly asks for validation. +Status: partially implemented. The crate now builds a dependency-free +`tinyjuice` binary with `reduce`, `reduce-json`, `verify`, `discover`, `wrap`, +`ls`, `cat`, `stats`, and `doctor` commands. +`reduce` accepts stdin or one file plus command metadata flags and prints +reduced inline text. `reduce-json` accepts the documented protocol payload from +stdin or one file, prints response JSON, and exits non-zero with structured +error JSON on invalid payloads. `verify --rules --fixtures` reports rule and +fixture diagnostics and exits non-zero for hard verification failures. +`discover` accepts a JSON array or NDJSON `ToolExecutionInput` stream and emits +metadata-only fallback family counts. `wrap` runs a command after `--`, reduces +captured stdout/stderr with exit-code metadata, and exits with the wrapped +command's status. `ls`, `cat`, and `stats` operate on an explicit CCR disk +tier via `--store-dir`; `cat` supports line and byte ranges, while `stats` +reports metadata-only token counts and byte totals. `doctor` emits structured +health JSON with `ok`, `warn`, `broken`, and `disabled` checks for built-in +rules, optional fixture verification, optional explicit CCR disk store +inspection, and host-aware `codex`, `openhuman`, and aggregate `hooks` targets. +`install codex` and `uninstall codex` maintain a TinyJuice-managed Codex +instruction block idempotently while preserving unrelated text. OpenHuman and +other host install/uninstall mutation remains a future CLI slice. + ## P1: Rule Validation And Discovery Plan: @@ -122,10 +161,20 @@ Acceptance: - Validation is deterministic and does not panic on bad user files. - Discovery reports command families and counts without raw tool output. +Status: implemented for the library surface. `verify_rules()` now reports parse +errors, invalid regex patterns, duplicate rule ids, and shadowed lower-priority +rules across the same builtin/user/project roots as `load_rules()`, without +changing the lenient runtime loader. Current diagnostics show the 101 built-ins +parse without duplicate ids and include 11 counter regexes that Rust `regex` +drops because they use lookahead. `verify_rule_fixtures()` walks +`*.fixture.json` examples, runs them through compiled rules, and reports pass +counts, parse errors, and hash-only output mismatches. +`discover_fallback_outputs()` groups command families that still classify to +`generic/fallback` while omitting raw tool output. + ## What Not To Do - Do not implement host installers before `doctor` can verify them. - Do not silently compact CI logs by default. - Do not turn every shell output into generic head/tail truncation. - Do not store raw artifacts unless explicitly requested. - diff --git a/prompt.md b/prompt.md index 4496e8b..434640c 100644 --- a/prompt.md +++ b/prompt.md @@ -105,11 +105,14 @@ compressor upgrades, P2 slices — all specified in the port plan. per-tool char cap → byte-cap backstop. TokenJuice may receive summarizer output, not raw output. - `AgentTokenjuiceCompression::Auto` is resolved host-side - (`agent/harness/definition.rs:487`: coding models → Light, else Full). The - crate's silent `Auto == Full` fallback in `options_for_agent()` should - become an explicit resolution requirement — do not rely on it. -- Two recovery tools are registered in OpenHuman with separate - implementations; consolidation is a Phase 3 task in the integration plan. + (`agent/harness/definition.rs:487`: coding models → Light, else Full). + TinyJuice and the mirrored OpenHuman copy now treat unresolved `Auto` as + passthrough with `none/agent-profile-auto-unresolved`; do not pass unresolved + `Auto` into the TokenJuice adapter. +- Two recovery tool names are still registered in OpenHuman during migration: + canonical `tokenjuice_retrieve` plus legacy `retrieve_tool_output`. The + legacy tool now delegates to the canonical implementation; remaining Phase 3 + work is migration cleanup, not behavior divergence. - `ml_compression_enabled` (Kompress bridge) stays default-off until tag protection fixtures exist. diff --git a/src/cache/mod.rs b/src/cache/mod.rs index 023494b..f0cb466 100644 --- a/src/cache/mod.rs +++ b/src/cache/mod.rs @@ -8,7 +8,8 @@ pub use marker::{ format_marker, is_recovery_tool, parse_markers, recovery_footer, }; pub use store::{ - DEFAULT_DISK_MAX_BYTES, DEFAULT_MAX_BYTES, DEFAULT_MAX_ENTRIES, GcStats, RangeUnit, configure, - configure_disk_cap, disable_disk_tier, enable_disk_tier, gc_disk_dir, gc_disk_tier, offload, - offload_checked, offload_checked_with_hash, retrieve, retrieve_range, short_hash, stats, + CcrPutResult, CcrStore, DEFAULT_DISK_MAX_BYTES, DEFAULT_MAX_BYTES, DEFAULT_MAX_ENTRIES, + GcStats, GlobalCcrStore, MemoryCcrStore, RangeUnit, configure, configure_disk_cap, + disable_disk_tier, enable_disk_tier, gc_disk_dir, gc_disk_tier, offload, offload_checked, + offload_checked_with_hash, retrieve, retrieve_range, short_hash, stats, }; diff --git a/src/cache/store.rs b/src/cache/store.rs index 42af7cb..d741923 100644 --- a/src/cache/store.rs +++ b/src/cache/store.rs @@ -39,6 +39,148 @@ const DISK_CAP_ENFORCE_EVERY: u64 = 128; /// capability token. const HASH_BYTES: usize = 16; +/// Result of attempting to store an original in CCR. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct CcrPutResult { + token: String, + retained: bool, +} + +impl CcrPutResult { + /// Build a put result. Invalid token shapes are never reported as retained. + pub fn new(token: String, retained: bool) -> Self { + let retained = retained && is_valid_token(&token); + Self { token, retained } + } + + /// CCR token derived for the original. + pub fn token(&self) -> &str { + &self.token + } + + /// True when the store retained the original and retrieval may be advertised. + pub fn retained(&self) -> bool { + self.retained + } +} + +/// Injectable CCR store. +pub trait CcrStore: Send + Sync { + fn put(&self, content: &str) -> CcrPutResult; + fn get(&self, token: &str) -> Option; + fn get_range(&self, token: &str, start: usize, end: usize, unit: RangeUnit) -> Option { + let original = self.get(token)?; + range_from_original(&original, start, end, unit) + } +} + +/// Compatibility wrapper over the process-global CCR store. +#[derive(Debug, Default, Clone, Copy)] +pub struct GlobalCcrStore; + +impl CcrStore for GlobalCcrStore { + fn put(&self, content: &str) -> CcrPutResult { + let (token, retained) = offload_checked(content); + CcrPutResult::new(token, retained) + } + + fn get(&self, token: &str) -> Option { + retrieve(token) + } + + fn get_range(&self, token: &str, start: usize, end: usize, unit: RangeUnit) -> Option { + retrieve_range(token, start, end, unit) + } +} + +/// Isolated in-memory CCR store for tests and host-managed adapters. +pub struct MemoryCcrStore { + inner: Mutex, + max_entries: usize, + max_bytes: usize, + ttl: Option, + disk_root: Option, +} + +impl Default for MemoryCcrStore { + fn default() -> Self { + Self::new(DEFAULT_MAX_ENTRIES, DEFAULT_MAX_BYTES) + } +} + +impl MemoryCcrStore { + pub fn new(max_entries: usize, max_bytes: usize) -> Self { + Self { + inner: Mutex::new(Inner::default()), + max_entries: max_entries.max(1), + max_bytes: max_bytes.max(1), + ttl: None, + disk_root: None, + } + } + + pub fn with_ttl(mut self, ttl: Option) -> Self { + self.ttl = ttl; + self + } + + pub fn with_disk_root(mut self, root: PathBuf) -> Self { + if std::fs::create_dir_all(&root).is_ok() { + self.disk_root = Some(root); + } + self + } + + pub fn stats(&self) -> (usize, usize) { + let inner = self.inner.lock().unwrap_or_else(|p| p.into_inner()); + (inner.map.len(), inner.total_bytes) + } +} + +impl CcrStore for MemoryCcrStore { + fn put(&self, content: &str) -> CcrPutResult { + let token = short_hash(content); + let mem_retained = self.inner.lock().unwrap_or_else(|p| p.into_inner()).insert( + token.clone(), + content.to_string(), + self.max_entries, + self.max_bytes, + ); + let mut disk_retained = false; + if let Some(root) = self.disk_root.as_ref() { + let path = root.join(&token); + if std::fs::write(path, content).is_ok() { + disk_retained = true; + } + } + CcrPutResult::new(token, mem_retained || disk_retained) + } + + fn get(&self, token: &str) -> Option { + if !is_valid_token(token) { + return None; + } + { + let mut inner = self.inner.lock().unwrap_or_else(|p| p.into_inner()); + if let Some(entry) = inner.map.get(token) { + if self.ttl.is_none_or(|ttl| entry.created.elapsed() < ttl) { + return Some(entry.content.clone()); + } + if let Some(entry) = inner.map.remove(token) { + inner.total_bytes = inner.total_bytes.saturating_sub(entry.content.len()); + } + } + } + let root = self.disk_root.as_ref()?; + let path = root.join(token); + if disk_entry_expired(&path, self.ttl) { + let _ = std::fs::remove_file(path); + return None; + } + std::fs::read_to_string(path).ok() + } +} + /// Tunable limits, settable once at startup from the `[tinyjuice]` config. struct Limits { max_entries: usize, @@ -419,14 +561,23 @@ pub enum RangeUnit { /// only when the original isn't available at all. pub fn retrieve_range(hash: &str, start: usize, end: usize, unit: RangeUnit) -> Option { let original = retrieve(hash)?; + range_from_original(&original, start, end, unit) +} + +fn range_from_original( + original: &str, + start: usize, + end: usize, + unit: RangeUnit, +) -> Option { if end <= start { return Some(String::new()); } match unit { RangeUnit::Bytes => { // Clamp to char boundaries so we never split a UTF-8 sequence. - let s = floor_char_boundary(&original, start.min(original.len())); - let e = floor_char_boundary(&original, end.min(original.len())); + let s = floor_char_boundary(original, start.min(original.len())); + let e = floor_char_boundary(original, end.min(original.len())); Some(original[s..e].to_string()) } RangeUnit::Lines => { @@ -585,6 +736,46 @@ mod tests { ); } + #[test] + fn memory_store_is_isolated_from_global_cache() { + let store = MemoryCcrStore::new(10, 10_000); + let original = "isolated memory store payload delta ".repeat(20); + let put = store.put(&original); + + assert!(put.retained()); + assert_eq!(store.get(put.token()).as_deref(), Some(original.as_str())); + assert_eq!(retrieve(put.token()), None, "global cache must not see it"); + } + + #[test] + fn memory_store_rejects_oversized_originals() { + let store = MemoryCcrStore::new(10, 16); + let original = "oversized isolated payload echo ".repeat(20); + let put = store.put(&original); + + assert!(!put.retained()); + assert_eq!(store.get(put.token()), None); + assert_eq!(store.stats(), (0, 0)); + } + + #[test] + fn memory_store_rejects_malformed_tokens_before_disk_lookup() { + let dir = std::env::temp_dir().join(format!( + "tj-memory-ccr-{}", + short_hash("memory-store-malformed-token") + )); + let _ = std::fs::remove_dir_all(&dir); + let store = MemoryCcrStore::new(10, 10_000).with_disk_root(dir.clone()); + + assert_eq!(store.get("../../state/config.toml"), None); + assert_eq!( + store.get_range("../../state/config.toml", 0, 1, RangeUnit::Lines), + None + ); + + let _ = std::fs::remove_dir_all(&dir); + } + #[test] fn disk_tier_survives_memory_miss() { let dir = std::env::temp_dir().join(format!("tj-ccr-{}", short_hash("disk-test-seed"))); diff --git a/src/cache_hints.rs b/src/cache_hints.rs new file mode 100644 index 0000000..03fe9d4 --- /dev/null +++ b/src/cache_hints.rs @@ -0,0 +1,229 @@ +use serde::{Deserialize, Serialize}; +use serde_json::{Map, Value}; +use sha2::{Digest, Sha256}; + +const STATIC_PREFIX_CACHE_KEY_PREFIX: &str = "tj-static-prefix-sha256:"; + +/// Provider-neutral prompt cache time-to-live hint. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub enum PromptCacheTtl { + Ephemeral, + OneHour, +} + +/// Provider-neutral location where an adapter may place a cache marker. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub enum CacheMarkerPlacement { + SystemPrompt, + Message, + ToolSchema, +} + +/// Cache marker routing hint. Core TinyJuice never mutates provider payloads. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct PromptCacheHint { + pub provider: String, + pub message_index: Option, + pub ttl: PromptCacheTtl, + pub placement: CacheMarkerPlacement, +} + +/// Static tool schema material used when deriving a stable prefix cache key. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ToolSchemaForCache { + pub name: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub kind: Option, + pub schema: Value, +} + +impl ToolSchemaForCache { + pub fn new(name: impl Into, schema: Value) -> Self { + Self { + name: name.into(), + kind: None, + schema, + } + } + + pub fn with_kind(mut self, kind: impl Into) -> Self { + self.kind = Some(kind.into()); + self + } +} + +/// Static-prefix routing metadata. `frozen_prefix_bytes` is an exact copy of +/// the caller-provided prefix bytes so adapters can splice it byte-identically. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct StaticPrefixCacheHint { + pub cache_key: String, + pub frozen_prefix_bytes: Vec, + pub tool_count: usize, +} + +/// Derive a stable cache key from static instructions plus sorted tool schemas. +pub fn stable_prefix_cache_key(instructions: &str, tools: &[ToolSchemaForCache]) -> String { + let mut hasher = Sha256::new(); + hasher.update(instructions.as_bytes()); + hasher.update([0]); + for tool in sorted_tool_schemas(tools) { + hasher.update(canonical_tool_schema(&tool).as_bytes()); + hasher.update([0]); + } + format!( + "{STATIC_PREFIX_CACHE_KEY_PREFIX}{}", + hex::encode(hasher.finalize()) + ) +} + +/// Build static-prefix metadata without rewriting the frozen prefix bytes. +pub fn static_prefix_cache_hint( + instructions: &str, + tools: &[ToolSchemaForCache], +) -> StaticPrefixCacheHint { + StaticPrefixCacheHint { + cache_key: stable_prefix_cache_key(instructions, tools), + frozen_prefix_bytes: instructions.as_bytes().to_vec(), + tool_count: tools.len(), + } +} + +/// Select Anthropic-style cache marker carriers as hints only: system prompt, +/// then the last three non-system messages. +pub fn anthropic_cache_hints(message_roles: &[impl AsRef]) -> Vec { + let mut hints = Vec::new(); + if !message_roles.is_empty() { + hints.push(PromptCacheHint { + provider: "anthropic".to_owned(), + message_index: Some(0), + ttl: PromptCacheTtl::Ephemeral, + placement: CacheMarkerPlacement::SystemPrompt, + }); + } + + let mut carriers: Vec = message_roles + .iter() + .enumerate() + .filter_map(|(idx, role)| (role.as_ref() != "system").then_some(idx)) + .collect(); + carriers.truncate(carriers.len().saturating_sub(3)); + let last_three_start = carriers.len(); + + for idx in message_roles + .iter() + .enumerate() + .filter_map(|(idx, role)| (role.as_ref() != "system").then_some(idx)) + .skip(last_three_start) + { + if hints.len() >= 4 { + break; + } + hints.push(PromptCacheHint { + provider: "anthropic".to_owned(), + message_index: Some(idx), + ttl: PromptCacheTtl::Ephemeral, + placement: CacheMarkerPlacement::Message, + }); + } + + hints +} + +fn sorted_tool_schemas(tools: &[ToolSchemaForCache]) -> Vec { + let mut sorted = tools.to_vec(); + sorted.sort_by(|a, b| { + a.name + .cmp(&b.name) + .then_with(|| a.kind.cmp(&b.kind)) + .then_with(|| canonical_json(&a.schema).cmp(&canonical_json(&b.schema))) + }); + sorted +} + +fn canonical_tool_schema(tool: &ToolSchemaForCache) -> String { + let mut map = Map::new(); + map.insert("kind".to_owned(), canonicalize_json(&tool.kind)); + map.insert("name".to_owned(), canonicalize_json(&tool.name)); + map.insert("schema".to_owned(), canonicalize_json(&tool.schema)); + canonical_json(&Value::Object(map)) +} + +fn canonicalize_json(value: &T) -> Value { + canonicalize_value(serde_json::to_value(value).unwrap_or(Value::Null)) +} + +fn canonicalize_value(value: Value) -> Value { + match value { + Value::Array(values) => Value::Array(values.into_iter().map(canonicalize_value).collect()), + Value::Object(map) => { + let mut entries: Vec<_> = map.into_iter().collect(); + entries.sort_by(|a, b| a.0.cmp(&b.0)); + Value::Object( + entries + .into_iter() + .map(|(key, value)| (key, canonicalize_value(value))) + .collect(), + ) + } + other => other, + } +} + +fn canonical_json(value: &Value) -> String { + serde_json::to_string(&canonicalize_value(value.clone())).unwrap_or_else(|_| "null".to_owned()) +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + #[test] + fn tool_order_does_not_change_static_prefix_cache_key() { + let tools = vec![ + ToolSchemaForCache::new("read_file", json!({"input": {"path": "string"}})), + ToolSchemaForCache::new("search", json!({"input": {"query": "string"}})), + ]; + let reversed = tools.iter().cloned().rev().collect::>(); + + assert_eq!( + stable_prefix_cache_key("instructions", &tools), + stable_prefix_cache_key("instructions", &reversed) + ); + } + + #[test] + fn canonical_json_key_order_does_not_change_cache_key() { + let left = ToolSchemaForCache::new("tool", json!({"b": 2, "a": {"d": 4, "c": 3}})); + let right = ToolSchemaForCache::new("tool", json!({"a": {"c": 3, "d": 4}, "b": 2})); + + assert_eq!( + stable_prefix_cache_key("instructions", &[left]), + stable_prefix_cache_key("instructions", &[right]) + ); + } + + #[test] + fn frozen_prefix_bytes_are_preserved_exactly() { + let instructions = "system prompt\nkeep bytes"; + let hint = static_prefix_cache_hint(instructions, &[]); + + assert_eq!(hint.frozen_prefix_bytes, instructions.as_bytes()); + assert!(hint.cache_key.starts_with(STATIC_PREFIX_CACHE_KEY_PREFIX)); + } + + #[test] + fn anthropic_hints_mark_system_and_last_three_carriers() { + let hints = anthropic_cache_hints(&["system", "user", "assistant", "tool", "assistant"]); + + assert_eq!(hints.len(), 4); + assert_eq!(hints[0].placement, CacheMarkerPlacement::SystemPrompt); + assert_eq!(hints[1].message_index, Some(2)); + assert_eq!(hints[3].message_index, Some(4)); + } +} diff --git a/src/compress.rs b/src/compress.rs index 198a384..3c9a5e1 100644 --- a/src/compress.rs +++ b/src/compress.rs @@ -12,12 +12,29 @@ //! [`CompressInput`] with a derived command/argv and calls [`route`]. use crate::cache; -use crate::compressors::{compressor_for, generic_compressor}; +use crate::cache::CcrStore; +use crate::compressors::{ + code::CodeStubTransform, + compressor_for, + diff::DiffNoiseTransform, + generic_compressor, + html::HtmlExtractTransform, + json::{SmartCrusherRowsTransform, SmartCrusherTableTransform}, + log::{LogTemplateTransform, SignalLogTransform}, + search::SearchTransform, + text::TextCrusherTransform, +}; use crate::detect::detect_content_kind; +use crate::pipeline::{ + BloatEstimate, OffloadTransform, PipelineInput, PipelineReport, PipelineSkipReason, + ReformatTransform, TypedPipelineOutput, estimate_bloat, run_typed_pipeline, +}; +use crate::policy::{ShellCompactionPolicy, ShellPolicyDecision, apply_shell_compaction_policy}; use crate::savings; use crate::tokens::estimate_tokens_with; use crate::types::{ CompressInput, CompressOptions, CompressOutput, CompressedOutput, ContentHint, ContentKind, + ReadIntent, }; /// Compress arbitrary content. Detects the kind (honouring `hint`), routes to @@ -31,6 +48,31 @@ pub async fn compress_content( hint: Option, opts: &CompressOptions, ) -> CompressedOutput { + compress_content_with_store_report(content, hint, opts, &cache::GlobalCcrStore) + .await + .0 +} + +/// Store-injected variant of [`compress_content`]. +pub async fn compress_content_with_store( + content: &str, + hint: Option, + opts: &CompressOptions, + store: &dyn CcrStore, +) -> CompressedOutput { + compress_content_with_store_report(content, hint, opts, store) + .await + .0 +} + +/// Store-injected variant of [`compress_content`] that also returns a redacted +/// pipeline report. +pub async fn compress_content_with_store_report( + content: &str, + hint: Option, + opts: &CompressOptions, + store: &dyn CcrStore, +) -> (CompressedOutput, PipelineReport) { let hint = hint.unwrap_or_default(); let input = CompressInput { content, @@ -41,30 +83,154 @@ pub async fn compress_content( argv: None, original_bytes: content.len(), }; - route(input, opts).await + route_with_store_report(input, opts, store).await } /// Core router: detect (unless the input already carries a resolved kind via the /// hint's explicit override), pick the compressor honouring config gates, run /// it, and apply CCR offload + footer. -pub async fn route(mut input: CompressInput<'_>, opts: &CompressOptions) -> CompressedOutput { +pub async fn route(input: CompressInput<'_>, opts: &CompressOptions) -> CompressedOutput { + route_with_store_report(input, opts, &cache::GlobalCcrStore) + .await + .0 +} + +/// Policy-aware variant of [`route`] for hosts that expose shell-policy config. +pub async fn route_with_shell_policy( + input: CompressInput<'_>, + opts: &CompressOptions, + shell_policy: ShellCompactionPolicy, +) -> CompressedOutput { + route_with_store_report_shell_policy(input, opts, &cache::GlobalCcrStore, shell_policy) + .await + .0 +} + +/// Store-injected variant of [`route`]. +pub async fn route_with_store( + input: CompressInput<'_>, + opts: &CompressOptions, + store: &dyn CcrStore, +) -> CompressedOutput { + route_with_store_report(input, opts, store).await.0 +} + +/// Store-injected, policy-aware variant of [`route`]. +pub async fn route_with_store_shell_policy( + input: CompressInput<'_>, + opts: &CompressOptions, + store: &dyn CcrStore, + shell_policy: ShellCompactionPolicy, +) -> CompressedOutput { + route_with_store_report_shell_policy(input, opts, store, shell_policy) + .await + .0 +} + +/// Store-injected variant of [`route`] that also returns a redacted pipeline +/// report. +pub async fn route_with_store_report( + input: CompressInput<'_>, + opts: &CompressOptions, + store: &dyn CcrStore, +) -> (CompressedOutput, PipelineReport) { + route_with_store_report_shell_policy( + input, + opts, + store, + ShellCompactionPolicy::AllowSafeInventory, + ) + .await +} + +/// Store-injected, policy-aware variant of [`route_with_store_report`]. +pub async fn route_with_store_report_shell_policy( + mut input: CompressInput<'_>, + opts: &CompressOptions, + store: &dyn CcrStore, + shell_policy: ShellCompactionPolicy, +) -> (CompressedOutput, PipelineReport) { let content = input.content; let original_bytes = content.len(); - // The global floor gates everything; the (lower) log floor lets small - // failure logs through. Below the smaller of the two, bail before paying - // for detection (which parses entire JSON blobs) — the kind is only a - // label on a passthrough. let log_floor = opts .min_bytes_to_compress_log .min(opts.min_bytes_to_compress); if !opts.router_enabled || original_bytes < log_floor { - let kind = input.hint.explicit.unwrap_or(ContentKind::PlainText); - return CompressedOutput::passthrough(content.to_string(), kind); + let kind = if opts.router_enabled { + input.hint.explicit.unwrap_or(ContentKind::PlainText) + } else { + detect_content_kind(content, input.hint) + }; + let res = CompressedOutput::passthrough(content.to_string(), kind); + let skip = if opts.router_enabled { + PipelineSkipReason::BelowMinBytes + } else { + PipelineSkipReason::RouterDisabled + }; + let report = PipelineReport::passthrough(kind, original_bytes, skip) + .with_bloat_estimate(estimate_bloat(content, kind)); + return (res, report); } let kind = detect_content_kind(content, input.hint); + let bloat_estimate = estimate_bloat(content, kind); + if is_exact_file_read(input.hint) { + let res = CompressedOutput::passthrough(content.to_string(), kind); + let report = PipelineReport::passthrough( + kind, + original_bytes, + PipelineSkipReason::Other("exact_file_read"), + ) + .with_bloat_estimate(bloat_estimate); + return (res, report); + } + let shell_policy_decision = apply_shell_compaction_policy( + &crate::types::ToolExecutionInput { + tool_name: input + .hint + .source_tool + .clone() + .unwrap_or_else(|| "shell".to_owned()), + command: input.command.clone(), + argv: input.argv.clone(), + stdout: Some(content.to_owned()), + exit_code: input.exit_code, + ..Default::default() + }, + shell_policy, + ); + if !matches!(shell_policy_decision, ShellPolicyDecision::Compact) { + let res = CompressedOutput::passthrough(content.to_string(), kind); + let report = PipelineReport::passthrough( + kind, + original_bytes, + PipelineSkipReason::Other(shell_policy_decision.as_str()), + ) + .with_bloat_estimate(bloat_estimate); + return (res, report); + } + if should_skip_low_bloat(&input, bloat_estimate.score) { + let res = CompressedOutput::passthrough(content.to_string(), kind); + let report = + PipelineReport::passthrough(kind, original_bytes, PipelineSkipReason::LowBloat) + .with_bloat_estimate(bloat_estimate); + return (res, report); + } input.kind = kind; + let original_tokens = estimate_tokens_with(content, opts.chars_per_token); + let ccr_for_typed = opts.ccr_enabled && original_tokens as usize >= opts.ccr_min_tokens; + + if let Some((res, report)) = try_typed_route( + &input, + opts, + store, + bloat_estimate, + original_tokens, + ccr_for_typed, + ) { + return (res, report); + } // Between the log floor and the global floor only log-like content // proceeds: detected logs, or command output headed for the rule engine. @@ -72,7 +238,14 @@ pub async fn route(mut input: CompressInput<'_>, opts: &CompressOptions) -> Comp if original_bytes < opts.min_bytes_to_compress { let log_like = kind == ContentKind::Log || input.command.is_some(); if !log_like || original_bytes < opts.min_bytes_to_compress_log { - return CompressedOutput::passthrough(content.to_string(), kind); + let res = CompressedOutput::passthrough(content.to_string(), kind); + let report = PipelineReport::passthrough( + kind, + original_bytes, + PipelineSkipReason::BelowMinBytes, + ) + .with_bloat_estimate(bloat_estimate); + return (res, report); } } @@ -99,62 +272,82 @@ pub async fn route(mut input: CompressInput<'_>, opts: &CompressOptions) -> Comp } let Some(out) = produced else { - return CompressedOutput::passthrough(content.to_string(), kind); + let res = CompressedOutput::passthrough(content.to_string(), kind); + let report = + PipelineReport::passthrough(kind, original_bytes, PipelineSkipReason::NoCompressor) + .with_bloat_estimate(bloat_estimate); + return (res, report); }; if out.text.len() >= original_bytes { - return CompressedOutput::passthrough(content.to_string(), kind); - } - - // CCR threshold: only offload when the input is large enough to be worth - // caching, or when the compression is heavily lossy on a non-trivial input - // (rationale on `CompressOptions::ccr_min_tokens`). When CCR is not in - // play a lossy result carries no recovery footer. `out.lossy` means the - // compressor *dropped* information (vs. a faithful reformat like a JSON - // table or HTML→text, which reports `lossy=false` and always ships). So - // when CCR can't back it up and the caller hasn't opted into - // `lossy_without_ccr`, we pass the original through untouched rather than - // emit a partial view the caller can never recover — the same guarantee - // that keeps code bodies, log lines, diff context, and sampled JSON rows - // whole in the no-CCR path. - let original_tokens = estimate_tokens_with(content, opts.chars_per_token); + let res = CompressedOutput::passthrough(content.to_string(), kind); + let report = + PipelineReport::passthrough(kind, original_bytes, PipelineSkipReason::NoSavings) + .with_bloat_estimate(bloat_estimate); + return (res, report); + } + let compacted_body_tokens = estimate_tokens_with(&out.text, opts.chars_per_token); let heavy_crush = compacted_body_tokens <= original_tokens / 2 && original_tokens as usize >= opts.ccr_min_tokens / 4; let ccr_for_call = opts.ccr_enabled && (original_tokens as usize >= opts.ccr_min_tokens || heavy_crush); if out.lossy && !ccr_for_call && !opts.lossy_without_ccr { - return CompressedOutput::passthrough(content.to_string(), kind); - } - - // Offload the original and append a recovery footer when CCR is in play. - let (text, ccr_token) = if ccr_for_call { - // The token is the content hash, so the footer is computable before - // storing anything. If the footer tips the result to at least the - // original size we pass through — checked first so the cache isn't - // left holding an entry no output references. - let token = cache::short_hash(content); - let footer = cache::recovery_footer(&token, original_bytes, out.lossy); - if out.text.len() + footer.len() >= original_bytes { - return CompressedOutput::passthrough(content.to_string(), kind); - } - let retained = cache::offload_checked_with_hash(&token, content); - if !retained { + let res = CompressedOutput::passthrough(content.to_string(), kind); + let report = + PipelineReport::passthrough(kind, original_bytes, PipelineSkipReason::CcrDisabled) + .with_bloat_estimate(bloat_estimate); + return (res, report); + } + + // Offload the original and expose the recovery footer separately. `text` + // below remains the compatibility output (body + footer), while hosts with + // their own caps can truncate `body` and reattach `recovery_footer`. + let (body, recovery_footer, ccr_token) = if ccr_for_call { + let put = store.put(content); + if !put.retained() { // The original is too large to keep in memory (over the byte cap) - // and the disk tier isn't on, so it can't be recovered. A lossy - // view is irreversible — decline it unless the caller allows - // unrecoverable lossy output. Either way no (dangling) recovery - // footer is attached. - if out.lossy && !opts.lossy_without_ccr { - return CompressedOutput::passthrough(content.to_string(), kind); + // and the disk tier isn't on, so it can't be recovered. A lossy view + // would be irreversible — decline it. A lossless reformat is still + // safe to return, just without a (dangling) recovery footer. + if out.lossy { + let res = CompressedOutput::passthrough(content.to_string(), kind); + let report = PipelineReport::passthrough( + kind, + original_bytes, + PipelineSkipReason::CcrNotRetained, + ) + .with_bloat_estimate(bloat_estimate); + return (res, report); } - (out.text, None) + (out.text, None, None) } else { - let mut text = out.text; + let token = put.token().to_string(); + let footer = cache::recovery_footer(&token, original_bytes, out.lossy); + let mut text = out.text.clone(); text.push_str(&footer); - (text, Some(token)) + // The footer adds bytes — if it tipped us over the original size, bail. + if text.len() >= original_bytes { + let res = CompressedOutput::passthrough(content.to_string(), kind); + let report = PipelineReport::passthrough( + kind, + original_bytes, + PipelineSkipReason::FooterWouldGrow, + ) + .with_bloat_estimate(bloat_estimate); + return (res, report); + } + (out.text, Some(footer), Some(token)) } } else { - (out.text, None) + (out.text, None, None) + }; + + let text = if let Some(footer) = recovery_footer.as_deref() { + let mut text = body.clone(); + text.push_str(footer); + text + } else { + body.clone() }; let compacted_bytes = text.len(); @@ -171,11 +364,24 @@ pub async fn route(mut input: CompressInput<'_>, opts: &CompressOptions) -> Comp ); // Record savings for the dashboard (tokens + cost saved for the LLM the - // result is being compressed for). - savings::record(kind, out.kind, original_tokens, compacted_tokens); + // result is being compressed for). Token counts are estimated; byte counts + // are measured directly from this reducer invocation. + savings::record_event( + savings::SavingsRecord::estimated_compaction( + kind, + out.kind, + original_tokens, + compacted_tokens, + ) + .with_bytes(original_bytes as u64, compacted_bytes as u64) + .with_lossy(out.lossy) + .with_ccr_token_present(ccr_token.is_some()), + ); - CompressedOutput { + let res = CompressedOutput { text, + body, + recovery_footer, content_kind: kind, compressor: out.kind, lossy: out.lossy, @@ -183,7 +389,213 @@ pub async fn route(mut input: CompressInput<'_>, opts: &CompressOptions) -> Comp ccr_token, original_bytes, compacted_bytes, + }; + let report = PipelineReport::applied( + kind, + original_bytes, + compacted_bytes, + out.kind, + out.lossy, + res.ccr_token.clone(), + ) + .with_bloat_estimate(bloat_estimate); + (res, report) +} + +fn try_typed_route( + input: &CompressInput<'_>, + opts: &CompressOptions, + store: &dyn CcrStore, + bloat_estimate: BloatEstimate, + original_tokens: u64, + ccr_for_call: bool, +) -> Option<(CompressedOutput, PipelineReport)> { + if has_command_context(input) { + return None; } + + let pipeline_input = PipelineInput::from(input); + let typed = match input.kind { + ContentKind::Json => { + let table = SmartCrusherTableTransform; + let rows = input + .hint + .query + .as_deref() + .filter(|query| !query.trim().is_empty()) + .map(|query| SmartCrusherRowsTransform::new().with_query(query)) + .unwrap_or_default(); + let reformats: [&dyn ReformatTransform; 1] = [&table]; + let offloads: Vec<&dyn OffloadTransform> = if ccr_for_call { + vec![&rows] + } else { + Vec::new() + }; + run_typed_pipeline(pipeline_input, &reformats, &offloads, store) + } + ContentKind::Diff => { + if !ccr_for_call { + return None; + } + let transform = DiffNoiseTransform::default(); + let reformats: [&dyn ReformatTransform; 0] = []; + let offloads: [&dyn OffloadTransform; 1] = [&transform]; + run_typed_pipeline(pipeline_input, &reformats, &offloads, store) + } + ContentKind::Log => { + let template = LogTemplateTransform; + let signal = SignalLogTransform; + let reformats: [&dyn ReformatTransform; 1] = [&template]; + let offloads: Vec<&dyn OffloadTransform> = if ccr_for_call { + vec![&signal] + } else { + Vec::new() + }; + run_typed_pipeline(pipeline_input, &reformats, &offloads, store) + } + ContentKind::Search => { + if !opts.search_enabled || !ccr_for_call { + return None; + } + let transform = input + .hint + .query + .as_deref() + .filter(|query| !query.trim().is_empty()) + .map(|query| SearchTransform::new().with_query(query)) + .unwrap_or_default(); + let reformats: [&dyn ReformatTransform; 0] = []; + let offloads: [&dyn OffloadTransform; 1] = [&transform]; + run_typed_pipeline(pipeline_input, &reformats, &offloads, store) + } + ContentKind::Html => { + if !opts.html_enabled || !ccr_for_call { + return None; + } + let transform = HtmlExtractTransform; + let reformats: [&dyn ReformatTransform; 0] = []; + let offloads: [&dyn OffloadTransform; 1] = [&transform]; + run_typed_pipeline(pipeline_input, &reformats, &offloads, store) + } + ContentKind::Code => { + if !opts.code_enabled || !ccr_for_call { + return None; + } + let ReadIntent::Stub(mode) = &input.hint.read_intent else { + return None; + }; + let mut transform = CodeStubTransform::new(mode.clone()); + if let Some(extension) = input.hint.extension.as_deref() { + transform = transform.with_extension(extension); + } + let reformats: [&dyn ReformatTransform; 0] = []; + let offloads: [&dyn OffloadTransform; 1] = [&transform]; + run_typed_pipeline(pipeline_input, &reformats, &offloads, store) + } + ContentKind::PlainText => { + if opts.ml_text_enabled || !ccr_for_call { + return None; + } + let transform = input + .hint + .query + .as_deref() + .filter(|query| !query.trim().is_empty()) + .map(|query| TextCrusherTransform::new(opts.clone()).with_query(query)) + .unwrap_or_else(|| TextCrusherTransform::new(opts.clone())); + let reformats: [&dyn ReformatTransform; 0] = []; + let offloads: [&dyn OffloadTransform; 1] = [&transform]; + run_typed_pipeline(pipeline_input, &reformats, &offloads, store) + } + }; + + finalize_typed_output(input, typed, bloat_estimate, original_tokens) +} + +fn finalize_typed_output( + input: &CompressInput<'_>, + typed: TypedPipelineOutput, + bloat_estimate: BloatEstimate, + original_tokens: u64, +) -> Option<(CompressedOutput, PipelineReport)> { + let original_bytes = input.content.len(); + if typed.report.applied_steps.is_empty() || typed.text.len() >= original_bytes { + return None; + } + + let (body, recovery_footer, ccr_token) = if typed.lossy { + let token = typed.ccr_token.clone()?; + let footer = cache::recovery_footer(&token, original_bytes, true); + let mut text = typed.text.clone(); + text.push_str(&footer); + if text.len() >= original_bytes { + let res = CompressedOutput::passthrough(input.content.to_string(), input.kind); + let report = PipelineReport::passthrough( + input.kind, + original_bytes, + PipelineSkipReason::FooterWouldGrow, + ) + .with_bloat_estimate(bloat_estimate); + return Some((res, report)); + } + (typed.text, Some(footer), Some(token)) + } else { + (typed.text, None, None) + }; + + let text = if let Some(footer) = recovery_footer.as_deref() { + let mut text = body.clone(); + text.push_str(footer); + text + } else { + body.clone() + }; + + let compacted_bytes = text.len(); + let compacted_tokens = estimate_tokens_with(&text, 4.0); + log::info!( + "[tokenjuice] compacted kind={} compressor={} lossy={} {}->{} bytes (~{}->{} tok)", + input.kind.as_str(), + typed.kind.as_str(), + typed.lossy, + original_bytes, + compacted_bytes, + original_tokens, + compacted_tokens, + ); + + savings::record_event( + savings::SavingsRecord::estimated_compaction( + input.kind, + typed.kind, + original_tokens, + compacted_tokens, + ) + .with_bytes(original_bytes as u64, compacted_bytes as u64) + .with_lossy(typed.lossy) + .with_ccr_token_present(ccr_token.is_some()), + ); + + let mut report = typed.report; + report.compacted_bytes = compacted_bytes; + report.bloat_estimate = Some(bloat_estimate); + report.ccr_tokens = ccr_token.clone().into_iter().collect(); + report.lossy = typed.lossy; + report.skip_reason = None; + + let res = CompressedOutput { + text, + body, + recovery_footer, + content_kind: input.kind, + compressor: typed.kind, + lossy: typed.lossy, + applied: true, + ccr_token, + original_bytes, + compacted_bytes, + }; + Some((res, report)) } /// Build a [`CompressorKind`] label-free passthrough quickly (used by callers @@ -192,9 +604,40 @@ pub fn detect_only(content: &str, hint: &ContentHint) -> ContentKind { detect_content_kind(content, hint) } +fn is_exact_file_read(hint: &ContentHint) -> bool { + matches!(hint.read_intent, crate::types::ReadIntent::Exact) + && hint + .source_tool + .as_deref() + .is_some_and(|tool| matches!(tool, "file_read" | "read_file" | "fs_read")) +} + +fn should_skip_low_bloat(input: &CompressInput<'_>, bloat_score: u8) -> bool { + if bloat_score > 0 { + return false; + } + if has_command_context(input) { + return false; + } + if input + .hint + .query + .as_deref() + .is_some_and(|query| !query.trim().is_empty()) + { + return false; + } + !matches!(input.hint.read_intent, crate::types::ReadIntent::Stub(_)) +} + +fn has_command_context(input: &CompressInput<'_>) -> bool { + input.command.is_some() || input.argv.as_ref().is_some_and(|argv| !argv.is_empty()) +} + #[cfg(test)] mod tests { use super::*; + use crate::ShellCompactionPolicy; use crate::types::CompressorKind; fn opts() -> CompressOptions { @@ -221,12 +664,353 @@ mod tests { let token = res.ccr_token.expect("offloaded"); assert_eq!(cache::retrieve(&token).as_deref(), Some(original.as_str())); assert!( - res.text.contains("⟦tj:"), + res.text.contains("tinyjuice_retrieve"), "footer marker present: {}", res.text ); } + #[tokio::test] + async fn diff_noise_output_offloads_original_for_recovery() { + let mut original = String::from("diff --git a/Cargo.lock b/Cargo.lock\n"); + original.push_str("@@ -1,800 +1,900 @@\n"); + for i in 0..700 { + original.push_str(&format!("+ new dependency entry {i} checksum {}\n", i * 17)); + } + for i in 0..350 { + original.push_str(&format!("- old dependency entry {i} checksum {}\n", i * 19)); + } + let hint = ContentHint { + explicit: Some(ContentKind::Diff), + ..Default::default() + }; + let store = cache::MemoryCcrStore::new(10, 1_000_000); + + let res = compress_content_with_store(&original, Some(hint), &opts(), &store).await; + + assert!(res.applied); + assert!(res.lossy); + assert_eq!(res.content_kind, ContentKind::Diff); + assert!(res.text.contains("reason=lockfile"), "{}", res.text); + let token = res.ccr_token.as_deref().expect("offloaded"); + assert_eq!(store.get(token).as_deref(), Some(original.as_str())); + } + + #[tokio::test] + async fn textcrusher_output_offloads_original_for_recovery() { + let mut original = String::new(); + for i in 0..80 { + original.push_str(&format!( + "routine service note {i} describes ordinary progress through the worker queue.\n\n" + )); + } + original.push_str( + "ERROR sync.worker.v2 failed for REQUEST_ID 9F42 after retry 17 in api.worker.example.\n\n", + ); + for i in 80..160 { + original.push_str(&format!( + "routine service note {i} describes ordinary progress through the worker queue.\n\n" + )); + } + let hint = ContentHint { + explicit: Some(ContentKind::PlainText), + query: Some("sync.worker.v2 REQUEST_ID".to_string()), + ..Default::default() + }; + let store = cache::MemoryCcrStore::new(10, 1_000_000); + let mut opts = opts(); + opts.ccr_min_tokens = 1; + + let res = compress_content_with_store(&original, Some(hint), &opts, &store).await; + + assert!(res.applied); + assert!(res.lossy); + assert_eq!(res.content_kind, ContentKind::PlainText); + assert_eq!(res.compressor, CompressorKind::TextCrusher); + assert!( + res.body.contains("ERROR sync.worker.v2 failed"), + "{}", + res.body + ); + let token = res.ccr_token.as_deref().expect("offloaded"); + assert_eq!(store.get(token).as_deref(), Some(original.as_str())); + } + + #[tokio::test] + async fn exposes_body_and_recovery_footer_separately() { + let mut rows = Vec::new(); + for i in 0..120 { + rows.push(format!( + r#"{{"id":{i},"name":"isolated_account_{i}","email":"iso{i}@ex.com","tier":"platinum"}}"# + )); + } + let original = format!("[{}]", rows.join(",")); + let res = compress_content(&original, None, &opts()).await; + + assert!(res.applied); + let footer = res + .recovery_footer + .as_deref() + .expect("offloaded output exposes footer"); + assert!( + !res.body.contains("tinyjuice_retrieve"), + "body must not contain footer" + ); + assert!( + footer.contains("tinyjuice_retrieve"), + "footer carries marker: {footer}" + ); + assert_eq!(res.text, format!("{}{}", res.body, footer)); + assert_eq!(res.compacted_bytes, res.text.len()); + } + + #[tokio::test] + async fn store_injected_route_uses_isolated_ccr_store() { + let store = cache::MemoryCcrStore::new(10, 1_000_000); + let mut rows = Vec::new(); + for i in 0..120 { + rows.push(format!( + r#"{{"id":{i},"name":"store_only_account_{i}","email":"store{i}@ex.com","tier":"silver"}}"# + )); + } + let original = format!("[{}]", rows.join(",")); + let res = compress_content_with_store(&original, None, &opts(), &store).await; + + assert!(res.applied); + let token = res.ccr_token.as_deref().expect("offloaded"); + assert_eq!(store.get(token).as_deref(), Some(original.as_str())); + assert_eq!(cache::retrieve(token), None, "global cache must not see it"); + } + + #[tokio::test] + async fn report_records_applied_step_and_ccr_token() { + let store = cache::MemoryCcrStore::new(10, 1_000_000); + let mut rows = Vec::new(); + for i in 0..120 { + rows.push(format!( + r#"{{"id":{i},"name":"report_account_{i}","email":"report{i}@ex.com","tier":"bronze"}}"# + )); + } + let original = format!("[{}]", rows.join(",")); + let (res, report) = + compress_content_with_store_report(&original, None, &opts(), &store).await; + + assert!(res.applied); + assert_eq!(report.content_kind, ContentKind::Json); + assert_eq!(report.original_bytes, original.len()); + assert_eq!(report.compacted_bytes, res.text.len()); + assert_eq!(report.applied_steps.len(), 1); + assert_eq!(report.applied_steps[0].name, "smartcrusher_rows"); + assert_eq!( + report.applied_steps[0].compressor, + Some(CompressorKind::SmartCrusher) + ); + assert_eq!( + report + .bloat_estimate + .map(|estimate| estimate.reason.as_str()), + Some("json_rows") + ); + assert_eq!(report.ccr_tokens, vec![res.ccr_token.unwrap()]); + assert_eq!(report.skip_reason, None); + } + + #[tokio::test] + async fn report_records_redacted_skip_reason() { + let mut o = opts(); + o.router_enabled = false; + let (res, report) = compress_content_with_store_report( + "sensitive raw payload", + None, + &o, + &cache::GlobalCcrStore, + ) + .await; + + assert!(!res.applied); + assert_eq!(report.skip_reason, Some(PipelineSkipReason::RouterDisabled)); + assert!(report.bloat_estimate.is_some()); + assert!(report.applied_steps.is_empty()); + assert!(report.ccr_tokens.is_empty()); + } + + #[tokio::test] + async fn report_records_low_bloat_skip_before_compressor_work() { + let content = (0..180) + .map(|i| format!("unique observation {i} has ordinary prose without repeated bulk.")) + .collect::>() + .join("\n"); + let hint = ContentHint { + explicit: Some(ContentKind::PlainText), + ..Default::default() + }; + let (res, report) = compress_content_with_store_report( + &content, + Some(hint), + &opts(), + &cache::GlobalCcrStore, + ) + .await; + + assert!(!res.applied); + assert_eq!(res.text, content); + assert_eq!(report.skip_reason, Some(PipelineSkipReason::LowBloat)); + assert_eq!( + report + .bloat_estimate + .map(|estimate| estimate.reason.as_str()), + Some("low_signal") + ); + assert!(report.applied_steps.is_empty()); + } + + #[tokio::test] + async fn shell_policy_keeps_exact_file_read_raw_even_with_extension_hint() { + let content = (0..120) + .map(|i| format!("fn generated_{i}() {{ println!(\"{i}\"); }}")) + .collect::>() + .join("\n"); + let hint = ContentHint { + source_tool: Some("shell".to_owned()), + extension: Some("rs".to_owned()), + ..Default::default() + }; + let input = CompressInput { + content: &content, + kind: ContentKind::PlainText, + hint: &hint, + exit_code: None, + command: Some("cat src/lib.rs".to_owned()), + argv: None, + original_bytes: content.len(), + }; + + let store = cache::MemoryCcrStore::new(10, 1_000_000); + let (res, report) = route_with_store_report(input, &opts(), &store).await; + + assert!(!res.applied); + assert_eq!(res.text, content); + assert_eq!( + report.skip_reason, + Some(PipelineSkipReason::Other("skip_file_content")) + ); + } + + #[tokio::test] + async fn shell_policy_skip_all_keeps_command_output_raw() { + let content = "line one\nline two\n".repeat(120); + let hint = ContentHint { + source_tool: Some("shell".to_owned()), + ..Default::default() + }; + let input = CompressInput { + content: &content, + kind: ContentKind::PlainText, + hint: &hint, + exit_code: None, + command: Some("rg --files".to_owned()), + argv: None, + original_bytes: content.len(), + }; + + let store = cache::MemoryCcrStore::new(10, 1_000_000); + let (res, report) = route_with_store_report_shell_policy( + input, + &opts(), + &store, + ShellCompactionPolicy::SkipAll, + ) + .await; + + assert!(!res.applied); + assert_eq!(res.text, content); + assert_eq!( + report.skip_reason, + Some(PipelineSkipReason::Other("skip_all")) + ); + } + + #[tokio::test] + async fn shell_policy_compact_all_allows_file_content_commands() { + let mut rows = Vec::new(); + for i in 0..120 { + rows.push(format!( + r#"{{"id":{i},"name":"account_{i}","email":"a{i}@ex.com","tier":"gold"}}"# + )); + } + let content = format!("[{}]", rows.join(",")); + let hint = ContentHint { + source_tool: Some("shell".to_owned()), + extension: Some("json".to_owned()), + ..Default::default() + }; + let input = CompressInput { + content: &content, + kind: ContentKind::PlainText, + hint: &hint, + exit_code: None, + command: Some("cat src/lib.rs".to_owned()), + argv: None, + original_bytes: content.len(), + }; + + let mut opts = opts(); + opts.ccr_min_tokens = 1; + let store = cache::MemoryCcrStore::new(10, 1_000_000); + let (res, report) = route_with_store_report_shell_policy( + input, + &opts, + &store, + ShellCompactionPolicy::CompactAll, + ) + .await; + + assert!(res.applied); + assert_eq!(res.content_kind, ContentKind::Json); + assert_ne!(res.text, content); + assert_eq!(report.skip_reason, None); + } + + #[tokio::test] + async fn file_read_hint_is_exact_by_default_even_with_code_extension() { + let content = (0..120) + .map(|i| format!("pub fn generated_{i}() {{ println!(\"{i}\"); }}")) + .collect::>() + .join("\n"); + let hint = ContentHint { + source_tool: Some("file_read".to_owned()), + extension: Some("rs".to_owned()), + ..Default::default() + }; + let store = cache::MemoryCcrStore::new(10, 1_000_000); + let (res, report) = + compress_content_with_store_report(&content, Some(hint), &opts(), &store).await; + + assert!(!res.applied); + assert_eq!(res.text, content); + assert_eq!( + report.skip_reason, + Some(PipelineSkipReason::Other("exact_file_read")) + ); + } + + #[tokio::test] + async fn lossy_output_declines_when_injected_store_cannot_retain() { + let store = cache::MemoryCcrStore::new(10, 16); + let mut rows = Vec::new(); + for i in 0..120 { + rows.push(format!( + r#"{{"id":{i},"name":"account_{i}","email":"a{i}@ex.com","tier":"gold"}}"# + )); + } + let original = format!("[{}]", rows.join(",")); + let res = compress_content_with_store(&original, None, &opts(), &store).await; + + assert!(!res.applied); + assert_eq!(res.text, original); + assert!(res.ccr_token.is_none()); + } + #[tokio::test] async fn small_input_passes_through() { let res = compress_content("tiny", None, &opts()).await; @@ -307,43 +1091,34 @@ mod tests { ); } - /// The default declines *information-dropping* lossy output when CCR can't - /// back it up — the whole point of the lossless-without-cache guarantee. + /// The default may still apply lossless reformats when CCR is disabled; it + /// only declines information-dropping lossy output that cannot be recovered. #[tokio::test] - async fn default_declines_lossy_without_ccr() { + async fn default_allows_lossless_without_ccr() { let mut o = opts(); o.ccr_enabled = false; assert!(!o.lossy_without_ccr, "default is strict"); let original = big_log(); let res = compress_content(&original, Some(log_hint()), &o).await; - assert!(!res.applied, "default must decline unrecoverable lossy: {}", res.text); - assert_eq!(res.text, original); + assert!(res.applied, "lossless reformat should apply: {}", res.text); + assert!(!res.lossy, "without CCR output must not drop data"); + assert!(res.ccr_token.is_none()); + assert!(res.text.len() < original.len()); } - /// A large JSON array with CCR off is reshaped into a full markdown table: - /// a faithful, lossless reshape (every row kept), so it ships even under the - /// strict default — the "markdown trick" the SmartCrusher uses without a - /// recovery cache. + /// A large JSON array with CCR off declines lossy row sampling under the + /// strict default, because the sampled-away rows would not be recoverable. #[tokio::test] - async fn json_is_a_full_lossless_table_without_ccr() { + async fn json_declines_lossy_sampling_without_ccr() { let mut o = opts(); o.ccr_enabled = false; assert!(!o.lossy_without_ccr, "strict default"); let original = big_json_array(); let res = compress_content(&original, None, &o).await; assert_eq!(res.content_kind, ContentKind::Json); - assert!(res.applied, "faithful table ships without CCR: {}", res.text); - assert!(!res.lossy, "full table is a faithful reshape"); - assert!(res.text.len() < original.len()); - assert!(res.ccr_token.is_none(), "reshape needs no recovery"); - // Every row's identity survives — nothing sampled away. - for i in [0, 59, 119] { - assert!( - res.text.contains(&format!("account_{i}")), - "row {i} kept: {}", - res.text - ); - } + assert!(!res.applied, "unrecoverable lossy view declined: {res:?}"); + assert_eq!(res.text, original); + assert!(res.ccr_token.is_none()); } /// A faithful, information-preserving reshape (HTML extraction here) is not @@ -368,7 +1143,11 @@ mod tests { }; let res = compress_content(&html, Some(hint), &o).await; assert_eq!(res.content_kind, ContentKind::Html); - assert!(res.applied, "faithful reshape ships without CCR: {}", res.text); + assert!( + res.applied, + "faithful reshape ships without CCR: {}", + res.text + ); assert!(!res.lossy, "extraction is information-preserving"); assert!(res.ccr_token.is_none(), "no recovery needed for a reshape"); // Every service line's text survives the reshape. @@ -447,7 +1226,11 @@ mod tests { assert!(res.text.contains("pub fn worker_0"), "signatures kept"); // Every collapsed body carries a per-block retrieval token. let tokens = cache::parse_markers(&res.text); - assert!(!tokens.is_empty(), "recoverable tokens present: {}", res.text); + assert!( + !tokens.is_empty(), + "recoverable tokens present: {}", + res.text + ); let body = cache::retrieve(&tokens[0]).expect("stored"); assert!(body.contains("tmp_0_15"), "full body recoverable: {body}"); } @@ -497,8 +1280,8 @@ mod tests { } #[tokio::test] - async fn small_json_below_global_floor_still_passes_through() { - // Non-log content keeps the historical 2048-byte floor. + async fn small_json_below_global_floor_can_reformat_losslessly() { + // Typed lossless reformats are allowed before the global lossy floor. let mut rows = Vec::new(); for i in 0..20 { rows.push(format!( @@ -508,8 +1291,10 @@ mod tests { let original = format!("[{}]", rows.join(",")); assert!(original.len() > 512 && original.len() < 2048); let res = compress_content(&original, None, &CompressOptions::default()).await; - assert!(!res.applied, "small JSON stays passthrough: {res:?}"); - assert_eq!(res.text, original); + assert!(res.applied, "small JSON can reformat: {res:?}"); + assert!(!res.lossy); + assert!(res.ccr_token.is_none()); + assert!(res.text.len() < original.len()); } #[tokio::test] @@ -522,7 +1307,7 @@ mod tests { } #[tokio::test] - async fn heavy_crush_below_ccr_min_tokens_still_offloads() { + async fn heavy_crush_below_ccr_min_tokens_can_apply_losslessly() { // ~470 estimated tokens — under the flat ccr_min_tokens (500) but well // over its quarter — crushed to less than half the tokens: the // ratio-aware gate must offload so the original stays retrievable. @@ -545,8 +1330,15 @@ mod tests { ); let res = compress_content(&original, None, &o).await; assert!(res.applied, "response: {res:?}"); - let token = res.ccr_token.expect("ratio-aware gate offloads"); - assert_eq!(cache::retrieve(&token).as_deref(), Some(original.as_str())); + if res.lossy { + let token = res.ccr_token.expect("lossy output must offload"); + assert_eq!(cache::retrieve(&token).as_deref(), Some(original.as_str())); + } else { + assert!( + res.ccr_token.is_none(), + "lossless output should not require recovery" + ); + } } #[tokio::test] diff --git a/src/compressors/code.rs b/src/compressors/code.rs index 21095a0..a69d6aa 100644 --- a/src/compressors/code.rs +++ b/src/compressors/code.rs @@ -1,46 +1,99 @@ -//! Source-code compressor — keep the declaration skeleton, collapse bodies. +//! Source-code compressor — keep signatures, collapse bodies. //! //! Inspired by Headroom's `CodeCompressor`. The goal is to keep the structural //! skeleton an agent needs to navigate a file — imports, type/function/class -//! signatures, top-level constants, and (crucially) the member signatures of -//! containers — while collapsing the deep executable bodies that dominate byte -//! count. +//! signatures, top-level constants — while collapsing the deep bodies that +//! dominate byte count. //! -//! This module ships the language-agnostic **declaration-skeleton heuristic**: -//! it walks the file tracking a stack of open blocks, remembering for each -//! whether it is a *container* (class/struct/impl/trait/interface/enum/ -//! namespace/module/extern) or a *function/other* block. A line is kept when it -//! is at top level, when it is a block-opening signature at any depth, or when -//! it is a member sitting directly inside a container (field, const, method -//! signature, doc comment). Free-function bodies collapse to a -//! `{ … N line(s) … }` placeholder; large bodies keep a head/tail skeleton so a -//! few real lines survive around the elision. A higher-fidelity tree-sitter path -//! (Rust/TS/JS/Python) is layered on and selected by language; until then every -//! language uses the heuristic. The router offloads each omitted block to CCR -//! for exact recovery. +//! This module currently ships the language-agnostic **brace-depth heuristic**: +//! lines at brace nesting depth 0–1 are kept; deeper bodies collapse to a +//! `{ … N lines … }` placeholder. Lines carrying error/TODO markers are always +//! kept. A higher-fidelity tree-sitter path (Rust/TS/Python) is layered on in a +//! follow-up slice and selected by language; until then every language uses the +//! heuristic. The router offloads the original to CCR for exact recovery. use async_trait::async_trait; use std::fmt::Write as _; -use super::{BLOCK_NOTE, Compressor, block_token}; -use crate::types::{CompressInput, CompressOptions, CompressOutput, CompressorKind}; +use super::Compressor; +use crate::cache::CcrStore; +use crate::pipeline::{OffloadOutput, OffloadTransform, PipelineInput, estimate_bloat}; +use crate::types::{ + CodeElision, CodeStubOutput, CompressInput, CompressOptions, CompressOutput, CompressorKind, + ContentKind, LineRange, ParseStatus, ReadIntent, StubMode, SymbolSummary, +}; -/// A body must have at least this many *interior* lines (not counting the brace -/// lines) before it is worth replacing with a placeholder. Small bodies stay -/// verbatim — the marker (~45 bytes) can be longer than the code it replaces. -pub const MIN_BODY_LINES_TO_COLLAPSE: usize = 8; +/// Bodies with more than this many collapsed lines get a placeholder; shorter +/// ones are kept verbatim (collapsing tiny bodies isn't worth the marker). +pub const MIN_BODY_LINES_TO_COLLAPSE: usize = 4; -/// Once a collapsed body has at least this many interior lines, keep a skeleton -/// (first two + last one interior lines) around the elided middle instead of a -/// single marker, so the reader still sees how the body opens and returns. -const SKELETON_MIN_INTERIOR: usize = 12; - -/// A collapse must save at least this many bytes over the placeholder marker, -/// otherwise it is a no-op inflation and the body is kept verbatim. -const MIN_BYTE_SAVINGS: usize = 16; +/// Markers that force a line to be kept even inside a deep body. +const KEEP_MARKERS: &[&str] = &["TODO", "FIXME", "XXX", "error", "panic", "unsafe"]; pub struct CodeCompressor; +/// Typed offload transform for explicit source-code stub reads. +#[derive(Debug, Clone)] +pub struct CodeStubTransform { + mode: StubMode, + extension: Option, +} + +impl CodeStubTransform { + pub fn new(mode: StubMode) -> Self { + Self { + mode, + extension: None, + } + } + + pub fn with_extension(mut self, extension: impl Into) -> Self { + self.extension = Some(extension.into()); + self + } + + pub fn mode(&self) -> &StubMode { + &self.mode + } + + pub fn extension(&self) -> Option<&str> { + self.extension.as_deref() + } +} + +impl OffloadTransform for CodeStubTransform { + fn name(&self) -> &'static str { + "code_stub" + } + + fn estimate_bloat(&self, input: &PipelineInput<'_>) -> f32 { + if input.content_kind != ContentKind::Code { + return 0.0; + } + f32::from(estimate_bloat(input.content, input.content_kind).score) / 100.0 + } + + fn apply(&self, input: &PipelineInput<'_>, store: &dyn CcrStore) -> Option { + if input.content_kind != ContentKind::Code { + return None; + } + let stub = stub_code( + input.content, + self.extension(), + &self.mode, + input.original_bytes, + ); + if stub.text.len() >= input.content.len() { + return None; + } + OffloadOutput::from_retained_put( + stub.text, + CompressorKind::Code, + store.put(input.original_content), + ) + } +} + #[async_trait] impl Compressor for CodeCompressor { fn kind(&self) -> CompressorKind { @@ -50,241 +103,89 @@ impl Compressor for CodeCompressor { async fn compress( &self, input: &CompressInput<'_>, - opts: &CompressOptions, + _opts: &CompressOptions, ) -> Option { - // Per-body retrieval tokens only make sense when the CCR store is on. - let per_body_tokens = opts.ccr_enabled; - // A query token in a function name or body pins that body open so the - // caller can still read the code they were looking for. - let query = input.hint.query.as_deref(); - // When set, collapse only enough (largest bodies first) to reach this - // output/input byte ratio; `None` collapses every eligible body. - let target_ratio = opts.code_target_ratio; + if let ReadIntent::Stub(mode) = &input.hint.read_intent { + let stub = stub_code( + input.content, + input.hint.extension.as_deref(), + mode, + input.original_bytes, + ); + if stub.text.len() < input.content.len() { + return Some(CompressOutput::lossy(stub.text, CompressorKind::Code)); + } + return None; + } + // Prefer the AST path when a grammar matches the file's language; fall // back to the language-agnostic heuristic otherwise (or when the AST // path doesn't shrink the content). #[cfg(feature = "tinyjuice-treesitter")] if let Some(ext) = input.hint.extension.as_deref() - && let Some(out) = - treesitter::compress(input.content, ext, per_body_tokens, query, target_ratio) + && let Some(out) = treesitter::compress(input.content, ext) { return Some(out); } - compress_heuristic(input.content, per_body_tokens, query, target_ratio) - } -} - -/// An eligible collapse, fed to [`select_collapses`] for budget selection. -struct Candidate { - /// Position in the caller's segment/body list. - idx: usize, - /// Byte size of the body's interior lines — the largest-first sort key. - interior_bytes: usize, - /// Bytes saved by collapsing this body (verbatim minus rendered size). - savings: usize, - /// Whether the rendered placeholder carries a retrieval token (the first - /// one also pays for the shared [`BLOCK_NOTE`] footer). - has_token: bool, -} - -/// Pick which eligible bodies actually collapse. With no target ratio every -/// candidate collapses (the historical behavior). With a target, candidates -/// collapse largest-interior-first until the projected output size reaches -/// `target_ratio × input_len`; the rest stay fully intact. `baseline` is the -/// projected output size with nothing collapsed. -fn select_collapses( - mut candidates: Vec, - baseline: usize, - input_len: usize, - target_ratio: Option, -) -> Vec { - let Some(ratio) = target_ratio else { - return candidates.into_iter().map(|c| c.idx).collect(); - }; - let target = (f64::from(ratio) * input_len as f64) as usize; - candidates.sort_by_key(|c| std::cmp::Reverse(c.interior_bytes)); - let mut projected = baseline; - let mut note_paid = false; - let mut chosen = Vec::new(); - for c in candidates { - if projected <= target { - break; - } - if c.has_token && !note_paid { - note_paid = true; - projected += BLOCK_NOTE.len(); - } - projected = projected.saturating_sub(c.savings); - chosen.push(c.idx); + compress_heuristic(input.content) } - chosen } -/// Language-agnostic declaration-skeleton compressor. Keeps top-level lines, -/// block-opening signatures at any depth, and members sitting directly inside a -/// container; collapses executable bodies. With `per_body_tokens`, each -/// collapsed body is offloaded to CCR and its placeholder carries a token that -/// retrieves exactly that body. `query` (when set) pins any body whose name or -/// text mentions a query token. Returns `None` if it wouldn't shrink the content. -pub fn compress_heuristic( - content: &str, - per_body_tokens: bool, - query: Option<&str>, - target_ratio: Option, -) -> Option { +/// Language-agnostic brace-depth compressor. Keeps lines at depth ≤ 1, collapses +/// deeper runs. Returns `None` if it wouldn't shrink the content. +pub fn compress_heuristic(content: &str) -> Option { let lines: Vec<&str> = content.lines().collect(); if lines.len() < 12 { return None; } - // Phase 1: walk the file into alternating kept-text and body segments. - // Each entry records whether the open block is a container (true) or a - // function/other block (false). Stack length is the brace-nesting depth. - let mut stack: Vec = Vec::new(); - // Classification for the next `{` that opens on its own line (next-line - // brace style, e.g. PHP/C#): set from the preceding declaration line. - let mut pending_container: Option = None; + let mut out = String::with_capacity(content.len() / 2 + 64); + let mut depth: i32 = 0; let mut collapsed: Vec<&str> = Vec::new(); - // The kept line that opened the body currently accumulating in `collapsed`. - let mut body_sig: Option<&str> = None; - let mut in_block_comment = false; - let mut segments: Vec = Vec::new(); - for line in &lines { - let was_in_comment = in_block_comment; - let (opens, closes) = brace_delta(line, &mut in_block_comment); - let t = line.trim(); - let start_depth = stack.len(); - let enclosing_container = *stack.last().unwrap_or(&false); - - // Blank lines: at container/top level they separate members and are - // kept; inside a body they stay glued to the body so it collapses as one. - if t.is_empty() && !was_in_comment { - if start_depth == 0 || enclosing_container { - flush_body( - &mut segments, - &mut collapsed, - per_body_tokens, - body_sig, - query, - ); - push_kept(&mut segments, line); - } else { - collapsed.push(line); - } - continue; + let flush = |out: &mut String, collapsed: &mut Vec<&str>| { + if collapsed.is_empty() { + return; } - - let decl_opener = !was_in_comment && opens > closes && is_decl_opener_line(t); - let keep = start_depth == 0 || decl_opener || (enclosing_container && is_member_line(t)); - - if keep { - flush_body( - &mut segments, - &mut collapsed, - per_body_tokens, - body_sig, - query, - ); - push_kept(&mut segments, line); - // This kept line is the signature for whatever body follows. - body_sig = Some(line); + if collapsed.len() >= MIN_BODY_LINES_TO_COLLAPSE { + let _ = writeln!(out, " {{ … {} line(s) … }}", collapsed.len()); } else { - collapsed.push(line); + for l in collapsed.iter() { + let _ = writeln!(out, "{l}"); + } } + collapsed.clear(); + }; - // Update the block stack. Signature/declaration lines that carry no - // brace arm `pending_container` for the `{` that opens next line. - if was_in_comment { - // Depth is tracked by brace_delta's comment state; nothing to push. - } else if opens > closes { - let n = opens - closes; - let is_cont = if is_decl_opener_line(t) { - is_container_line(t) - } else { - pending_container.unwrap_or(false) - }; - for k in 0..n { - stack.push(k == 0 && is_cont); - } - pending_container = None; - } else if closes > opens { - for _ in 0..(closes - opens) { - stack.pop(); - } - pending_container = None; - } else if opens == 0 { - // No braces on this line: it may be a declaration whose body brace - // lands on the next line. - if is_container_line(t) { - pending_container = Some(true); - } else if is_signature_line(t) { - pending_container = Some(false); - } + for line in &lines { + let (opens, closes) = brace_delta(line); + let start_depth = depth; + // A line that carries signal is always kept, regardless of depth. + let force_keep = KEEP_MARKERS.iter().any(|m| line.contains(m)); + + // Keep top-level lines (depth 0) — imports, signatures, the line that + // opens a block — and collapse the block body (depth ≥ 1). Short bodies + // (e.g. small struct field lists) stay verbatim via the flush threshold, + // so struct/enum fields survive while long function bodies collapse. + if start_depth == 0 || force_keep { + flush(&mut out, &mut collapsed); + let _ = writeln!(out, "{line}"); } else { - // Balanced braces (one-liner block): depth unchanged. - pending_container = None; + collapsed.push(line); } - } - flush_body( - &mut segments, - &mut collapsed, - per_body_tokens, - body_sig, - query, - ); - - // Phase 2: pick which eligible bodies collapse. Baseline = the output size - // if every body stayed verbatim. - let baseline: usize = segments - .iter() - .map(|s| match s { - Segment::Kept(text) => text.len(), - Segment::Body(b) => b.verbatim.len(), - }) - .sum(); - let candidates: Vec = segments - .iter() - .enumerate() - .filter_map(|(idx, s)| match s { - Segment::Body(b) => b.collapsed.as_ref().map(|c| Candidate { - idx, - interior_bytes: b.interior_bytes, - savings: b.verbatim.len().saturating_sub(c.text.len()), - has_token: c.has_token, - }), - Segment::Kept(_) => None, - }) - .collect(); - let mut collapse_flags = vec![false; segments.len()]; - for idx in select_collapses(candidates, baseline, content.len(), target_ratio) { - collapse_flags[idx] = true; - } - - let mut out = String::with_capacity(content.len() / 2 + 64); - let mut any_token = false; - for (idx, seg) in segments.iter().enumerate() { - match seg { - Segment::Kept(text) => out.push_str(text), - Segment::Body(b) => match &b.collapsed { - Some(c) if collapse_flags[idx] => { - any_token |= c.has_token; - out.push_str(&c.text); - } - _ => out.push_str(&b.verbatim), - }, + depth += opens - closes; + if depth < 0 { + depth = 0; } } + flush(&mut out, &mut collapsed); - let mut out = out.trim_end().to_string(); - if any_token { - out.push_str(BLOCK_NOTE); - } + let out = out.trim_end().to_string(); if out.len() >= content.len() { return None; } log::debug!( - "[tinyjuice][code] heuristic {} -> {} bytes ({} lines)", + "[tokenjuice][code] heuristic {} -> {} bytes ({} lines)", content.len(), out.len(), lines.len() @@ -292,428 +193,413 @@ pub fn compress_heuristic( Some(CompressOutput::lossy(out, CompressorKind::Code)) } -/// One stretch of output from the phase-1 walk: kept lines, or a body run that -/// may collapse. -enum Segment { - Kept(String), - Body(BodySeg), -} +/// Build a structural source-code stub with elision metadata. +/// +/// Hosts call this directly for explicit stub reads. Exact reads should not use +/// this path. +pub fn stub_code( + content: &str, + extension: Option<&str>, + mode: &StubMode, + max_bytes: usize, +) -> CodeStubOutput { + #[cfg(feature = "tinyjuice-treesitter")] + if let Some(ext) = extension + && let Some(out) = treesitter::stub(content, ext, mode) + { + return enforce_stub_budget(out, max_bytes); + } -/// A body run with both renderings: verbatim (always available) and the -/// collapsed placeholder when the body is eligible. -struct BodySeg { - verbatim: String, - collapsed: Option, - /// Byte size of the interior lines — the largest-first selection key. - interior_bytes: usize, + enforce_stub_budget(stub_heuristic(content, mode), max_bytes) } -/// The rendered collapse of an eligible body. -struct CollapsedRender { - text: String, - has_token: bool, +#[derive(Debug, Clone)] +struct StubBody { + declaration_start: usize, + declaration_range: LineRange, + body_start: usize, + body_end: usize, + body_range: LineRange, + symbol: SymbolSummary, } -/// Append a kept line to the trailing kept segment (starting one if needed). -fn push_kept(segments: &mut Vec, line: &str) { - if let Some(Segment::Kept(text)) = segments.last_mut() { - let _ = writeln!(text, "{line}"); - } else { - segments.push(Segment::Kept(format!("{line}\n"))); +fn should_expand_body(mode: &StubMode, body: &StubBody) -> bool { + match mode { + StubMode::SignaturesOnly | StubMode::PublicApi => false, + StubMode::MatchedSymbols(symbols) => symbols.iter().any(|s| { + let wanted = s.trim(); + !wanted.is_empty() && wanted == body.symbol.name + }), + StubMode::ExpandAroundLines(ranges) => ranges + .iter() + .copied() + .any(|range| range.intersects(body.body_range)), } } -/// Close the accumulated body run (if any) into a body segment and clear it. -fn flush_body( - segments: &mut Vec, - collapsed: &mut Vec<&str>, - per_body_tokens: bool, - sig: Option<&str>, - query: Option<&str>, -) { - if collapsed.is_empty() { - return; +fn placeholder_for_body(content: &str, body: &StubBody) -> String { + let body_text = &content[body.body_start..body.body_end]; + let lines = body.body_range.end.saturating_sub(body.body_range.start) + 1; + if body_text.trim_start().starts_with('{') { + format!( + "{{ /* ... lines {}-{} elided ({lines} line(s)) ... */ }}", + body.body_range.start, body.body_range.end + ) + } else { + format!( + "... # lines {}-{} elided ({lines} line(s))", + body.body_range.start, body.body_range.end + ) } - segments.push(Segment::Body(render_body( - collapsed, - per_body_tokens, - sig, - query, - ))); - collapsed.clear(); } -/// Render an accumulated body run: verbatim when it is short, worth less than -/// the marker, or pinned by `keep_body`; otherwise also a placeholder rendering -/// the caller may substitute. Large bodies keep a head/tail skeleton around the -/// elided middle. `sig` is the signature line that opened the body (used to -/// read its name for the keep heuristics). -fn render_body( - collapsed: &[&str], - per_body_tokens: bool, - sig: Option<&str>, - query: Option<&str>, -) -> BodySeg { - // Interior = the run minus a leading bare `{` and a trailing `}` line. - let lo = usize::from(brace_open_only(collapsed[0].trim())); - let mut hi = collapsed.len(); - if hi > lo && brace_close_only(collapsed[hi - 1].trim()) { - hi -= 1; - } - let interior = &collapsed[lo..hi]; - let interior_count = interior.len(); - let interior_bytes: usize = interior.iter().map(|l| l.len() + 1).sum(); - - let name = sig.and_then(fn_name); - let body_text = collapsed.join("\n"); - let mut verbatim = String::with_capacity(body_text.len() + 1); - for l in collapsed { - let _ = writeln!(verbatim, "{l}"); - } - - let collapse = interior_count >= MIN_BODY_LINES_TO_COLLAPSE - && !keep_body(name.as_deref(), &body_text, interior_count, query); - - let mut rendered = None; - if collapse { - let indent = leading_ws(interior.first().copied().unwrap_or(collapsed[0])); - // The token recovers the FULL original body regardless of skeleton. - let token = block_token(&body_text, per_body_tokens); - - if interior_count >= SKELETON_MIN_INTERIOR { - let elided = interior_count - 3; - let marker = format!("{indent}{{ … {elided} line(s) …{token} }}"); - let removed: usize = interior[2..interior_count - 1] - .iter() - .map(|l| l.len() + 1) - .sum(); - if removed > marker.len() + MIN_BYTE_SAVINGS { - let mut text = String::new(); - let _ = writeln!(text, "{}", interior[0]); - let _ = writeln!(text, "{}", interior[1]); - let _ = writeln!(text, "{marker}"); - let _ = writeln!(text, "{}", interior[interior_count - 1]); - rendered = Some(CollapsedRender { - text, - has_token: !token.is_empty(), - }); - } +fn build_stub_from_bodies( + content: &str, + bodies: Vec, + mode: &StubMode, + parse_status: ParseStatus, +) -> CodeStubOutput { + let mut out = String::with_capacity(content.len() / 2 + 128); + let mut cursor = 0usize; + let mut symbols = Vec::new(); + let mut elisions = Vec::new(); + + for body in bodies { + if body.body_start < cursor { + continue; + } + if matches!(mode, StubMode::PublicApi) && !body.symbol.public { + out.push_str(&content[cursor..body.declaration_start]); + let lines = body + .body_range + .end + .saturating_sub(body.declaration_range.start) + + 1; + let _ = write!( + out, + "/* ... private declaration lines {}-{} elided ({lines} line(s)) ... */", + body.declaration_range.start, body.body_range.end + ); + elisions.push(CodeElision { + start_line: body.declaration_range.start, + end_line: body.body_range.end, + reason: "public_api_private_declaration".to_string(), + }); + cursor = body.body_end; + continue; + } + symbols.push(body.symbol.clone()); + out.push_str(&content[cursor..body.body_start]); + let body_text = &content[body.body_start..body.body_end]; + let body_lines = body_text.lines().count(); + if body_lines < MIN_BODY_LINES_TO_COLLAPSE || should_expand_body(mode, &body) { + out.push_str(body_text); } else { - let marker = format!("{indent}{{ … {} line(s) …{token} }}", collapsed.len()); - if body_text.len() > marker.len() + MIN_BYTE_SAVINGS { - rendered = Some(CollapsedRender { - text: format!("{marker}\n"), - has_token: !token.is_empty(), - }); - } + out.push_str(&placeholder_for_body(content, &body)); + elisions.push(CodeElision { + start_line: body.body_range.start, + end_line: body.body_range.end, + reason: format!("body_stub:{:?}", mode), + }); } + cursor = body.body_end; } + out.push_str(&content[cursor..]); - BodySeg { - verbatim, - collapsed: rendered, - interior_bytes, + CodeStubOutput { + text: out.trim_end().to_string(), + symbols, + elisions, + parse_status, } } -/// Leading-whitespace prefix of a line (for aligning placeholder markers). -fn leading_ws(s: &str) -> &str { - &s[..s.len() - s.trim_start().len()] -} - -/// A line that is nothing but an opening brace (next-line brace style). -fn brace_open_only(t: &str) -> bool { - t == "{" -} +fn stub_heuristic(content: &str, mode: &StubMode) -> CodeStubOutput { + let mut out = String::with_capacity(content.len() / 2 + 128); + let mut depth: i32 = 0; + let mut collapsed: Vec<(usize, &str)> = Vec::new(); + let mut elisions = Vec::new(); + let mut symbols = Vec::new(); + let mut skipped_private_start: Option = None; + let mut expanding_body = false; + + let flush = + |out: &mut String, collapsed: &mut Vec<(usize, &str)>, elisions: &mut Vec| { + if collapsed.is_empty() { + return; + } + if collapsed.len() >= MIN_BODY_LINES_TO_COLLAPSE { + let start = collapsed.first().map(|(n, _)| *n).unwrap_or(1); + let end = collapsed.last().map(|(n, _)| *n).unwrap_or(start); + let _ = writeln!( + out, + " {{ /* ... lines {start}-{end} elided ({} line(s)) ... */ }}", + collapsed.len() + ); + elisions.push(CodeElision { + start_line: start, + end_line: end, + reason: format!("heuristic_body_stub:{:?}", mode), + }); + } else { + for (_, line) in collapsed.iter() { + let _ = writeln!(out, "{line}"); + } + } + collapsed.clear(); + }; + let flush_verbatim = |out: &mut String, collapsed: &mut Vec<(usize, &str)>| { + for (_, line) in collapsed.iter() { + let _ = writeln!(out, "{line}"); + } + collapsed.clear(); + }; -/// A line that is only closing punctuation: `}`, `};`, `},`, `})`, `});` … -fn brace_close_only(t: &str) -> bool { - t.starts_with('}') - && t.chars() - .all(|c| matches!(c, '}' | ';' | ',' | ')' | ' ' | '\t')) -} + for (idx, line) in content.lines().enumerate() { + let line_no = idx + 1; + let (opens, closes) = brace_delta(line); + let start_depth = depth; + let force_keep = KEEP_MARKERS.iter().any(|m| line.contains(m)); + let mut next_depth = depth + opens - closes; + if next_depth < 0 { + next_depth = 0; + } -/// Keywords that introduce a *container* whose members we keep visible. -const CONTAINER_KW: &[&str] = &[ - "class", - "struct", - "impl", - "trait", - "interface", - "enum", - "namespace", - "module", - "mod", - "record", - "union", - "protocol", - "object", - "extension", -]; - -/// Keywords that introduce a function/method body. -const FN_KW: &[&str] = &["fn", "func", "def", "function", "fun", "sub", "method"]; - -/// Leading modifiers/visibility keywords to skip before the "kind" token. -const MODIFIER_KW: &[&str] = &[ - "pub", - "public", - "private", - "protected", - "internal", - "static", - "final", - "abstract", - "override", - "async", - "export", - "default", - "virtual", - "inline", - "unsafe", - "const", - "extern", - "sealed", - "partial", - "open", - "suspend", - "explicit", - "readonly", - "friend", - "data", - "mut", -]; - -/// Control-flow keywords whose `{` opens a block that is NOT a declaration. -const CONTROL_KW: &[&str] = &[ - "if", - "else", - "for", - "while", - "switch", - "do", - "try", - "catch", - "finally", - "match", - "loop", - "foreach", - "using", - "lock", - "fixed", - "synchronized", - "with", - "when", - "return", - "elif", - "except", -]; - -/// The token identifying a declaration's kind — the first word after any -/// leading visibility/modifier keywords. Trailing `{` and generics are ignored. -fn kind_token(t: &str) -> Option<&str> { - let mut toks = t.split(|c: char| c.is_whitespace() || c == '(' || c == '<' || c == '{'); - for tok in toks.by_ref() { - let w = tok.trim_end_matches(':'); - if w.is_empty() { + if let Some(start_line) = skipped_private_start { + depth = next_depth; + if depth == 0 { + elisions.push(CodeElision { + start_line, + end_line: line_no, + reason: "public_api_private_declaration".to_string(), + }); + let _ = writeln!( + out, + "/* ... private declaration lines {start_line}-{line_no} elided ({} line(s)) ... */", + line_no.saturating_sub(start_line) + 1 + ); + skipped_private_start = None; + } continue; } - if MODIFIER_KW.contains(&w) { + + if expanding_body { + let _ = writeln!(out, "{line}"); + depth = next_depth; + if depth == 0 { + expanding_body = false; + } continue; } - return Some(w); + + if start_depth == 0 || force_keep { + flush(&mut out, &mut collapsed, &mut elisions); + if let Some(symbol) = symbol_from_signature(line, line_no, line_no, "heuristic") { + if matches!(mode, StubMode::PublicApi) && !symbol.public { + if next_depth == 0 { + elisions.push(CodeElision { + start_line: line_no, + end_line: line_no, + reason: "public_api_private_declaration".to_string(), + }); + let _ = writeln!( + out, + "/* ... private declaration line {line_no} elided ... */" + ); + } else { + skipped_private_start = Some(line_no); + } + depth = next_depth; + continue; + } + if should_expand_heuristic_symbol(mode, &symbol, line_no) && next_depth > 0 { + expanding_body = true; + } + symbols.push(symbol); + } + let _ = writeln!(out, "{line}"); + } else { + if should_expand_heuristic_line(mode, line_no) { + flush_verbatim(&mut out, &mut collapsed); + let _ = writeln!(out, "{line}"); + expanding_body = next_depth > 0; + depth = next_depth; + continue; + } + collapsed.push((line_no, line)); + } + depth = next_depth; } - None -} + if let Some(start_line) = skipped_private_start { + let end_line = content.lines().count().max(start_line); + elisions.push(CodeElision { + start_line, + end_line, + reason: "public_api_private_declaration".to_string(), + }); + let _ = writeln!( + out, + "/* ... private declaration lines {start_line}-{end_line} elided ({} line(s)) ... */", + end_line.saturating_sub(start_line) + 1 + ); + } + flush(&mut out, &mut collapsed, &mut elisions); -/// True when the line introduces a container (its members should stay visible). -fn is_container_line(t: &str) -> bool { - if t.starts_with("extern") { - return true; + CodeStubOutput { + text: out.trim_end().to_string(), + symbols, + elisions, + parse_status: ParseStatus::HeuristicFallback, } - kind_token(t).is_some_and(|w| CONTAINER_KW.contains(&w)) } -/// True when a line that ENDS with `{` is a declaration opener (a container or -/// a function/method signature) rather than a control-flow block. -fn is_decl_opener_line(t: &str) -> bool { - let tr = t.trim_end(); - let head = match tr.strip_suffix('{') { - Some(h) => h.trim_end(), - None => return false, - }; - if head.is_empty() { - return false; // bare `{` - } - if is_container_line(head) { - return true; - } - match kind_token(head) { - Some(w) if CONTROL_KW.contains(&w) => false, - Some(w) if FN_KW.contains(&w) => true, - // `type name(params)` — Java/C#/C++/PHP method signature. - _ => head.contains('(') && head.contains(')'), +fn should_expand_heuristic_symbol(mode: &StubMode, symbol: &SymbolSummary, line_no: usize) -> bool { + match mode { + StubMode::MatchedSymbols(symbols) => symbols.iter().any(|s| { + let wanted = s.trim(); + !wanted.is_empty() && wanted == symbol.name + }), + StubMode::ExpandAroundLines(ranges) => ranges + .iter() + .copied() + .any(|range| range.intersects(LineRange::new(line_no, line_no))), + StubMode::SignaturesOnly | StubMode::PublicApi => false, } } -/// True when a line (carrying no brace) looks like a function/method signature -/// whose body brace lands on the next line. -fn is_signature_line(t: &str) -> bool { - if !t.contains('(') { - return false; - } - let tr = t.trim_end(); - // A trailing `;` is a statement or an abstract/interface method — no body. - if tr.ends_with(';') || tr.ends_with(',') { - return false; +fn should_expand_heuristic_line(mode: &StubMode, line_no: usize) -> bool { + match mode { + StubMode::ExpandAroundLines(ranges) => ranges + .iter() + .copied() + .any(|range| range.intersects(LineRange::new(line_no, line_no))), + StubMode::SignaturesOnly | StubMode::PublicApi | StubMode::MatchedSymbols(_) => false, } - !kind_token(t).is_some_and(|w| CONTROL_KW.contains(&w)) } -/// True when a line sitting directly inside a container is a member worth -/// keeping (field, const, doc comment, method signature) — anything that is not -/// a bare brace or empty. -fn is_member_line(t: &str) -> bool { - !t.is_empty() && !brace_open_only(t) && !brace_close_only(t) -} +fn enforce_stub_budget(mut stub: CodeStubOutput, max_bytes: usize) -> CodeStubOutput { + if max_bytes == 0 || stub.text.len() <= max_bytes { + return stub; + } -/// Extract the declared name from a signature line: the identifier immediately -/// before the first parameter list. Handles Go receiver methods. -fn fn_name(sig: &str) -> Option { - let s = sig.trim(); - // Strip a Go receiver: `func (r *T) Name(` -> `Name(`. - let s = if let Some(rest) = s.strip_prefix("func ") { - let rest = rest.trim_start(); - if rest.starts_with('(') { - rest.find(')').map(|i| rest[i + 1..].trim()).unwrap_or(rest) - } else { - rest + let marker = "\n[TOKENJUICE CODE STUB TRUNCATED]\n"; + let budget = max_bytes.saturating_sub(marker.len()).max(marker.len()); + let mut out = String::new(); + let mut kept_lines = 0usize; + for line in stub.text.lines() { + let addition = line.len() + 1; + if out.len() + addition + marker.len() > budget { + break; } - } else { - s - }; - let paren = s.find('(')?; - let before = &s[..paren]; - let name = before - .rsplit(|c: char| !c.is_alphanumeric() && c != '_' && c != '$') - .next() - .unwrap_or(""); - if name.is_empty() { - None - } else { - Some(name.to_string()) + out.push_str(line); + out.push('\n'); + kept_lines += 1; } + out.push_str(marker); + let total_lines = stub.text.lines().count(); + if kept_lines < total_lines { + stub.elisions.push(CodeElision { + start_line: kept_lines.saturating_add(1), + end_line: total_lines, + reason: "stub_budget".to_string(), + }); + } + stub.text = out.trim_end().to_string(); + stub } -/// Count word-boundary occurrences of `needle` inside the already-lowercased -/// `hay`. `needle` must itself be lowercase. -fn count_word(hay: &str, needle: &str) -> usize { - let mut n = 0; - let mut from = 0; - while let Some(rel) = hay[from..].find(needle) { - let start = from + rel; - let end = start + needle.len(); - let before_ok = start == 0 - || !hay[..start] - .chars() - .next_back() - .is_some_and(|c| c.is_alphanumeric() || c == '_'); - let after_ok = end >= hay.len() - || !hay[end..] - .chars() +fn symbol_from_signature( + signature: &str, + start_line: usize, + end_line: usize, + fallback_kind: &str, +) -> Option { + let trimmed = signature.trim(); + if trimmed.is_empty() { + return None; + } + + let patterns = [ + ("fn ", "function"), + ("function ", "function"), + ("def ", "function"), + ("class ", "class"), + ("struct ", "struct"), + ("enum ", "enum"), + ("trait ", "trait"), + ("interface ", "interface"), + ("impl ", "impl"), + ("const ", "const"), + ("let ", "binding"), + ("var ", "binding"), + ]; + let lower = trimmed.to_ascii_lowercase(); + for (needle, kind) in patterns { + if let Some(idx) = lower.find(needle) { + let rest = &trimmed[idx + needle.len()..]; + let name = rest + .trim_start() + .trim_start_matches("r#") + .split(|c: char| !(c.is_ascii_alphanumeric() || c == '_' || c == '$')) .next() - .is_some_and(|c| c.is_alphanumeric() || c == '_'); - if before_ok && after_ok { - n += 1; + .unwrap_or_default(); + if !name.is_empty() { + return Some(SymbolSummary { + name: name.to_string(), + kind: kind.to_string(), + start_line, + end_line, + public: is_public_signature(trimmed), + }); + } } - from = end; } - n -} -/// True when a body is error-handling dense: at least two of throw/raise/ -/// panic!/return Err/catch/except/rescue. -fn is_error_dense(body: &str) -> bool { - let l = body.to_ascii_lowercase(); - let mut hits = count_word(&l, "throw"); - hits += count_word(&l, "raise"); - hits += count_word(&l, "catch"); - hits += count_word(&l, "except"); - hits += count_word(&l, "rescue"); - // Punctuation-carrying / multi-word forms: plain substring counts. - hits += l.matches("panic!").count(); - hits += l.matches("return err").count(); - hits >= 2 + if fallback_kind != "heuristic" { + Some(SymbolSummary { + name: fallback_kind.to_string(), + kind: fallback_kind.to_string(), + start_line, + end_line, + public: is_public_signature(trimmed), + }) + } else { + None + } } -/// True when a function name or body mentions any (≥2-char) token from `query`. -fn query_hits(name: Option<&str>, body: &str, query: &str) -> bool { - let body_l = body.to_ascii_lowercase(); - let name_l = name.map(|n| n.to_ascii_lowercase()); - query - .split(|c: char| !c.is_alphanumeric() && c != '_') - .filter(|tok| tok.len() >= 2) - .any(|tok| { - let tl = tok.to_ascii_lowercase(); - body_l.contains(&tl) || name_l.as_deref().is_some_and(|n| n.contains(&tl)) - }) +fn is_public_signature(signature: &str) -> bool { + let trimmed = signature.trim_start(); + trimmed.starts_with("pub ") + || trimmed.starts_with("pub(") + || trimmed.starts_with("export ") + || trimmed.starts_with("public ") } -/// Decide whether a body is important enough to keep verbatim rather than -/// collapse: constructors/entrypoints, short accessors, error-dense bodies, or -/// bodies matching the caller's query. -fn keep_body(name: Option<&str>, body: &str, interior: usize, query: Option<&str>) -> bool { - if let Some(n) = name { - let nl = n.to_ascii_lowercase(); - if matches!( - nl.as_str(), - "main" | "new" | "init" | "__init__" | "__construct" | "constructor" - ) { - return true; - } - let is_accessor = nl.starts_with("get") - || nl.starts_with("set") - || nl.starts_with("is") - || nl.starts_with("has"); - if is_accessor && interior <= 6 { - return true; +fn line_starts(content: &str) -> Vec { + let mut starts = vec![0usize]; + for (idx, byte) in content.bytes().enumerate() { + if byte == b'\n' { + starts.push(idx + 1); } } - if is_error_dense(body) { - return true; - } - if let Some(q) = query - && query_hits(name, body, q) - { - return true; - } - false + starts } -/// Count `{`/`}` on a line, ignoring those inside string/char literals, -/// line comments, and `/* … */` block comments (Javadoc is full of literal -/// braces via `{@link …}` — counting them drifts the depth for the rest of -/// the file). `in_block_comment` carries block-comment state across lines. -fn brace_delta(line: &str, in_block_comment: &mut bool) -> (usize, usize) { - // A char literal ('x', '\n', '{') closes within a few characters; a Rust - // lifetime ('a in generics) or a stray apostrophe never does. Treating - // every ' as a string opener would flip the rest of a lifetime-carrying - // line into phantom string mode and stop counting its braces. - const CHAR_LITERAL_WINDOW: usize = 10; - - let mut opens = 0usize; - let mut closes = 0usize; +fn byte_to_line(starts: &[usize], byte: usize) -> usize { + starts.partition_point(|start| *start <= byte).max(1) +} + +/// Count `{`/`}` (and `(`/`)`) on a line, ignoring those inside string/char +/// literals and line comments — a cheap approximation good enough for the +/// depth heuristic. +fn brace_delta(line: &str) -> (i32, i32) { + let mut opens = 0i32; + let mut closes = 0i32; let mut in_str: Option = None; let mut prev = '\0'; - let mut iter = line.char_indices().peekable(); - while let Some((i, c)) = iter.next() { - if *in_block_comment { - if c == '/' && prev == '*' { - *in_block_comment = false; - } - prev = c; - continue; - } + let mut chars = line.chars().peekable(); + while let Some(c) = chars.next() { match in_str { Some(q) => { if c == q && prev != '\\' { @@ -721,30 +607,9 @@ fn brace_delta(line: &str, in_block_comment: &mut bool) -> (usize, usize) { } } None => match c { - '"' | '`' => in_str = Some(c), - '/' if iter.peek().map(|&(_, ch)| ch) == Some('*') => { - iter.next(); - *in_block_comment = true; - prev = '\0'; - continue; - } - '\'' => { - let rest = &line[i + c.len_utf8()..]; - let close = rest - .char_indices() - .take_while(|(off, _)| *off <= CHAR_LITERAL_WINDOW) - .find(|&(_, ch)| ch == '\'') - .map(|(off, _)| off); - if let Some(off) = close { - // Real char literal: skip past it wholesale so a - // brace inside (e.g. '{') isn't counted. - let end = i + c.len_utf8() + off; - while iter.next_if(|&(j, _)| j <= end).is_some() {} - } - // No close nearby: a lifetime/apostrophe — ignore it. - } - '/' if iter.peek().map(|&(_, ch)| ch) == Some('/') => break, // line comment - '#' => break, // python/shell comment + '"' | '\'' | '`' => in_str = Some(c), + '/' if chars.peek() == Some(&'/') => break, // line comment + '#' => break, // python/shell comment '{' => opens += 1, '}' => closes += 1, _ => {} @@ -758,14 +623,13 @@ fn brace_delta(line: &str, in_block_comment: &mut bool) -> (usize, usize) { /// AST-aware code compression via tree-sitter (Rust/TS/JS/Python). Keeps full /// source but replaces function/method bodies longer than a threshold with a /// `{ … N lines … }` (or `...` for Python) placeholder, preserving signatures, -/// imports, type declarations and struct/enum fields exactly. Constructors, -/// entrypoints, short accessors, error-dense bodies and query-matching bodies -/// are pinned open; large bodies keep a head/tail skeleton. +/// imports, type declarations and struct/enum fields exactly. #[cfg(feature = "tinyjuice-treesitter")] mod treesitter { use super::{ - BLOCK_NOTE, Candidate, CompressOutput, CompressorKind, MIN_BODY_LINES_TO_COLLAPSE, - SKELETON_MIN_INTERIOR, block_token, keep_body, leading_ws, select_collapses, + CompressOutput, CompressorKind, LineRange, MIN_BODY_LINES_TO_COLLAPSE, ParseStatus, + StubBody, StubMode, SymbolSummary, build_stub_from_bodies, byte_to_line, line_starts, + symbol_from_signature, }; use tree_sitter::{Node, Parser}; @@ -799,115 +663,58 @@ mod treesitter { "generator_function_declaration", ]; - /// A collapsible body plus the name of its enclosing function (when known). - struct Body { - start: usize, - end: usize, - name: Option, - } - - pub fn compress( - content: &str, - ext: &str, - per_body_tokens: bool, - query: Option<&str>, - target_ratio: Option, - ) -> Option { + pub fn compress(content: &str, ext: &str) -> Option { let (language, braced) = language_for(ext)?; let mut parser = Parser::new(); parser.set_language(&language).ok()?; let tree = parser.parse(content, None)?; let src = content.as_bytes(); - // Collect outermost collapsible bodies. - let mut bodies: Vec = Vec::new(); - collect_bodies(tree.root_node(), src, &mut bodies); - bodies.sort_by_key(|b| b.start); - let mut merged: Vec = Vec::new(); - for b in bodies { + // Collect outermost collapsible body byte-ranges. + let mut ranges: Vec<(usize, usize)> = Vec::new(); + collect_bodies(tree.root_node(), src, &mut ranges); + // Sort and drop nested ranges (keep outermost only). + ranges.sort_by_key(|r| r.0); + let mut merged: Vec<(usize, usize)> = Vec::new(); + for r in ranges { if let Some(last) = merged.last() - && b.start < last.end + && r.0 < last.1 { continue; // nested inside a body we're already collapsing } - merged.push(b); + merged.push(r); } if merged.is_empty() { return None; } - // Phase 1: render the collapse for every eligible body. - let renders: Vec> = merged - .iter() - .map(|b| { - let body = &content[b.start..b.end]; - let n_lines = body.lines().count(); - // Interior excludes the brace lines for braced bodies. - let interior = if braced { - n_lines.saturating_sub(2) - } else { - n_lines - }; - let collapse = interior >= MIN_BODY_LINES_TO_COLLAPSE - && !keep_body(b.name.as_deref(), body, interior, query); - if !collapse { - return None; - } - let token = block_token(body, per_body_tokens); - let rendered = render_collapse(body, braced, interior, n_lines, &token); - Some((rendered, !token.is_empty())) - }) - .collect(); - - // Phase 2: pick which eligible bodies collapse (largest interior - // first when a target ratio is set). Baseline = the full content, i.e. - // the output size if nothing collapsed. - let candidates: Vec = merged - .iter() - .zip(&renders) - .enumerate() - .filter_map(|(idx, (b, render))| { - render.as_ref().map(|(text, has_token)| Candidate { - idx, - interior_bytes: interior_bytes(&content[b.start..b.end], braced), - savings: (b.end - b.start).saturating_sub(text.len()), - has_token: *has_token, - }) - }) - .collect(); - let mut collapse_flags = vec![false; merged.len()]; - for idx in select_collapses(candidates, content.len(), content.len(), target_ratio) { - collapse_flags[idx] = true; - } - let mut out = String::with_capacity(content.len()); let mut cursor = 0usize; - let mut any_token = false; - for (idx, b) in merged.iter().enumerate() { - if b.start < cursor { + for (start, end) in merged { + if start < cursor { continue; } - out.push_str(&content[cursor..b.start]); - match &renders[idx] { - Some((rendered, has_token)) if collapse_flags[idx] => { - any_token |= *has_token; - out.push_str(rendered); - } - _ => out.push_str(&content[b.start..b.end]), + out.push_str(&content[cursor..start]); + let body = &content[start..end]; + let n_lines = body.lines().count(); + if n_lines < MIN_BODY_LINES_TO_COLLAPSE { + out.push_str(body); + } else if braced { + out.push_str(&format!("{{ … {n_lines} line(s) … }}")); + } else { + // Python suite — keep an indented ellipsis so it still reads. + out.push_str(&format!("... # {n_lines} line(s) collapsed")); } - cursor = b.end; + cursor = end; } out.push_str(&content[cursor..]); - let mut out = out.trim_end().to_string(); - if any_token { - out.push_str(BLOCK_NOTE); - } + let out = out.trim_end().to_string(); if out.len() >= content.len() { return None; } log::debug!( - "[tinyjuice][code] tree-sitter ext={} {} -> {} bytes", + "[tokenjuice][code] tree-sitter ext={} {} -> {} bytes", ext, content.len(), out.len() @@ -915,75 +722,41 @@ mod treesitter { Some(CompressOutput::lossy(out, CompressorKind::Code)) } - /// Byte size of a body's interior lines (excluding the brace lines for - /// braced bodies) — the largest-first selection key. - fn interior_bytes(body: &str, braced: bool) -> usize { - if braced { - let lines: Vec<&str> = body.lines().collect(); - if lines.len() < 3 { - return 0; - } - lines[1..lines.len() - 1].iter().map(|l| l.len() + 1).sum() - } else { - body.len() + pub fn stub(content: &str, ext: &str, mode: &StubMode) -> Option { + let (language, _braced) = language_for(ext)?; + let mut parser = Parser::new(); + parser.set_language(&language).ok()?; + let tree = parser.parse(content, None)?; + if tree.root_node().has_error() { + return None; } - } - /// Render a collapsed body: a head/tail skeleton for large bodies, a single - /// placeholder otherwise. The `token` recovers the FULL original body. - fn render_collapse( - body: &str, - braced: bool, - interior: usize, - n_lines: usize, - token: &str, - ) -> String { - if braced { - let lines: Vec<&str> = body.lines().collect(); - if interior >= SKELETON_MIN_INTERIOR && lines.len() >= 5 { - let inner = &lines[1..lines.len() - 1]; - let indent = leading_ws(inner[0]); - let elided = inner.len() - 3; - return format!( - "{{\n{}\n{}\n{indent}{{ … {elided} line(s) …{token} }}\n{}\n}}", - inner[0], - inner[1], - inner[inner.len() - 1], - ); - } - format!("{{ … {n_lines} line(s) …{token} }}") - } else { - // Python suite — indented ellipsis so it still reads. - let lines: Vec<&str> = body.lines().collect(); - if interior >= SKELETON_MIN_INTERIOR && lines.len() >= 4 { - let indent = leading_ws(lines[0]); - let elided = lines.len() - 3; - return format!( - "{}\n{}\n{indent}... # {elided} line(s) collapsed{token}\n{}", - lines[0], - lines[1], - lines[lines.len() - 1], - ); - } - format!("... # {n_lines} line(s) collapsed{token}") + let src = content.as_bytes(); + let starts = line_starts(content); + let mut bodies = Vec::new(); + collect_stub_bodies(tree.root_node(), content, src, &starts, &mut bodies); + bodies.sort_by_key(|body| body.body_start); + bodies.dedup_by_key(|body| (body.body_start, body.body_end)); + if bodies.is_empty() { + return None; } + + Some(build_stub_from_bodies( + content, + bodies, + mode, + ParseStatus::TreeSitter, + )) } - /// Recursively collect the byte-ranges (and names) of function/method bodies. - fn collect_bodies(node: Node, src: &[u8], out: &mut Vec) { + /// Recursively collect the byte-ranges of function/method bodies. + fn collect_bodies(node: Node, src: &[u8], out: &mut Vec<(usize, usize)>) { if BODY_PARENTS.contains(&node.kind()) && let Some(body) = node.child_by_field_name("body") { - let name = node - .child_by_field_name("name") - .and_then(|n| n.utf8_text(src).ok()) - .map(|s| s.to_string()); - out.push(Body { - start: body.start_byte(), - end: body.end_byte(), - name, - }); + out.push((body.start_byte(), body.end_byte())); // Don't descend into a collapsed body. + let _ = src; return; } let mut cursor = node.walk(); @@ -991,12 +764,62 @@ mod treesitter { collect_bodies(child, src, out); } } + + fn collect_stub_bodies( + node: Node, + content: &str, + src: &[u8], + starts: &[usize], + out: &mut Vec, + ) { + if BODY_PARENTS.contains(&node.kind()) + && let Some(body) = node.child_by_field_name("body") + { + let body_start = body.start_byte(); + let body_end = body.end_byte(); + let declaration_start = node.start_byte(); + let declaration_start_line = byte_to_line(starts, declaration_start); + let declaration_end_line = byte_to_line(starts, body_start); + let start_line = byte_to_line(starts, body_start); + let end_line = byte_to_line(starts, body_end.saturating_sub(1)); + let symbol = symbol_for_node(content, node, body, starts); + out.push(StubBody { + declaration_start, + declaration_range: LineRange::new(declaration_start_line, declaration_end_line), + body_start, + body_end, + body_range: LineRange::new(start_line, end_line), + symbol, + }); + let _ = src; + return; + } + let mut cursor = node.walk(); + for child in node.children(&mut cursor) { + collect_stub_bodies(child, content, src, starts, out); + } + } + + fn symbol_for_node(content: &str, node: Node, body: Node, starts: &[usize]) -> SymbolSummary { + let signature = &content[node.start_byte()..body.start_byte()]; + let start_line = byte_to_line(starts, node.start_byte()); + let end_line = byte_to_line(starts, body.start_byte()); + symbol_from_signature(signature, start_line, end_line, node.kind()).unwrap_or_else(|| { + SymbolSummary { + name: node.kind().to_string(), + kind: node.kind().to_string(), + start_line, + end_line, + public: false, + } + }) + } } #[cfg(test)] mod tests { use super::*; - use crate::cache; + use crate::cache::{CcrStore, MemoryCcrStore}; #[test] fn keeps_signatures_collapses_bodies() { @@ -1009,7 +832,7 @@ mod tests { } src.push_str(" tmp_0\n}\n\n"); src.push_str("struct Config {\n name: String,\n size: usize,\n}\n"); - let out = compress_heuristic(&src, false, None, None).expect("compresses"); + let out = compress_heuristic(&src).expect("compresses"); assert!(out.lossy); assert!( out.text.contains("pub fn process"), @@ -1032,64 +855,7 @@ mod tests { #[test] fn short_file_passes_through() { let src = "fn a() {}\nfn b() {}\n"; - assert!(compress_heuristic(src, false, None, None).is_none()); - } - - /// A large body keeps a head/tail skeleton: the first two and the last - /// interior lines survive, only the middle is elided. - #[test] - fn skeleton_keeps_head_and_tail_lines() { - let mut src = String::from("pub fn walk(items: &[i32]) -> i32 {\n"); - src.push_str(" let head_a = items.len();\n"); - src.push_str(" let head_b = items.first().copied().unwrap_or(0);\n"); - for i in 0..20 { - src.push_str(&format!(" let mid_{i} = compute(items, {i});\n")); - } - src.push_str(" head_a + head_b\n}\n"); - // Pad so the file clears the 12-line minimum with unrelated top-level lines. - for i in 0..4 { - src.push_str(&format!("pub const K_{i}: i32 = {i};\n")); - } - let out = compress_heuristic(&src, false, None, None).expect("compresses"); - assert!( - out.text.contains("let head_a = items.len();"), - "{}", - out.text - ); - assert!( - out.text.contains("let head_b = items.first"), - "second head line kept: {}", - out.text - ); - assert!( - out.text.contains("head_a + head_b"), - "tail line kept: {}", - out.text - ); - assert!(!out.text.contains("mid_10"), "middle elided: {}", out.text); - assert!(out.text.contains("line(s) …")); - } - - /// The full body is recoverable from the per-block token even when only a - /// skeleton is shown. - #[test] - fn skeleton_token_recovers_full_body() { - let mut src = String::from("pub fn scan(items: &[i32]) -> i32 {\n"); - src.push_str(" let a = items.len();\n"); - src.push_str(" let b = 0;\n"); - for i in 0..20 { - src.push_str(&format!(" let mid_{i} = compute(items, {i});\n")); - } - src.push_str(" a + b\n}\n"); - for i in 0..4 { - src.push_str(&format!("pub const C_{i}: i32 = {i};\n")); - } - let out = compress_heuristic(&src, true, None, None).expect("compresses"); - let tokens = cache::parse_markers(&out.text); - assert_eq!(tokens.len(), 1, "{}", out.text); - let body = cache::retrieve(&tokens[0]).expect("stored"); - assert!(body.contains("let mid_10 = compute(items, 10);"), "{body}"); - assert!(body.contains("a + b"), "full body recovered: {body}"); + assert!(compress_heuristic(src).is_none()); } #[cfg(feature = "tinyjuice-treesitter")] @@ -1104,7 +870,7 @@ mod tests { } src.push_str(" tmp_0\n}\n\n"); src.push_str("pub struct Config {\n pub name: String,\n pub size: usize,\n}\n"); - let out = treesitter::compress(&src, "rs", false, None, None).expect("compresses"); + let out = treesitter::compress(&src, "rs").expect("compresses"); assert!( out.text.contains("pub fn process(items: &[i32]) -> i32"), "{}", @@ -1127,604 +893,250 @@ mod tests { src.push_str(&format!(" x_{i} = compute(event, {i})\n")); } src.push_str(" return x_0\n"); - let out = treesitter::compress(&src, "py", false, None, None).expect("compresses"); + let out = treesitter::compress(&src, "py").expect("compresses"); assert!(out.text.contains("def handler(event):"), "{}", out.text); assert!(out.text.contains("collapsed"), "{}", out.text); assert!(!out.text.contains("x_15")); } - #[cfg(feature = "tinyjuice-treesitter")] #[test] - fn treesitter_keeps_main() { - let mut src = String::from("fn helper(items: &[i32]) -> i32 {\n"); + fn keeps_marker_lines_in_body() { + let mut src = String::from("fn f() {\n"); for i in 0..20 { - src.push_str(&format!(" let h_{i} = items.len() + {i};\n")); - } - src.push_str(" h_0\n}\n\n"); - src.push_str("fn main() {\n"); - for i in 0..20 { - src.push_str(&format!(" println!(\"step {i}\");\n")); + if i == 10 { + src.push_str(" // TODO: handle the edge case here\n"); + } else { + src.push_str(&format!(" do_thing({i});\n")); + } } - src.push_str(" helper(&[]);\n}\n"); - let out = treesitter::compress(&src, "rs", false, None, None).expect("compresses"); - // main is an entrypoint — it must survive verbatim. - assert!(out.text.contains("println!(\"step 10\")"), "{}", out.text); - // helper's body still collapses. - assert!(!out.text.contains("h_10"), "{}", out.text); - } - - #[test] - fn brace_delta_handles_lifetimes_and_char_literals() { - // Lifetimes must not open phantom string mode. - assert_eq!( - brace_delta("pub fn f<'a>(x: &'a str) {", &mut false), - (1, 0) - ); - assert_eq!( - brace_delta("impl<'de> Deserialize<'de> for X {", &mut false), - (1, 0) - ); - assert_eq!( - brace_delta( - "fn g<'a, 'b>(x: &'a str, y: &'b str) -> &'a str {", - &mut false - ), - (1, 0) - ); - // Real char literals still shield their contents. - assert_eq!(brace_delta("let c = '{';", &mut false), (0, 0)); - assert_eq!(brace_delta("if c == '}' { close(); }", &mut false), (1, 1)); - // Strings still shield braces. - assert_eq!(brace_delta(r#"let s = "{}"; f(|| {"#, &mut false), (1, 0)); + src.push_str("}\n"); + let out = compress_heuristic(&src).expect("compresses"); + assert!(out.text.contains("TODO"), "marker line kept:\n{}", out.text); } #[test] - fn lifetime_heavy_rust_keeps_signatures() { - let mut src = String::from("use serde::Deserialize;\n\n"); - src.push_str("pub fn parse<'de>(map: &Map) -> Config {\n"); + fn stub_code_reports_elisions_and_symbols() { + let mut src = String::from("use std::fmt;\n\npub fn visible() {\n"); for i in 0..20 { - src.push_str(&format!(" let field_{i} = map.next_value();\n")); + src.push_str(&format!(" println!(\"{i}\");\n")); } - src.push_str(" Config::default()\n}\n\n"); - src.push_str("pub fn lookup<'a>(map: &'a Map, key: &str) -> Option<&'a str> {\n"); - for i in 0..20 { - src.push_str(&format!(" let probe_{i} = map.get(key);\n")); - } - src.push_str(" None\n}\n\n"); - src.push_str("pub struct Cursor<'a> {\n buf: &'a [u8],\n pos: usize,\n}\n"); - let out = compress_heuristic(&src, false, None, None).expect("compresses"); - // Both signatures after the first lifetime-carrying line must survive - // at top level — depth drift used to collapse them. - assert!(out.text.contains("pub fn lookup<'a>"), "{}", out.text); - assert!(out.text.contains("pub struct Cursor<'a>"), "{}", out.text); - assert!( - !out.text.contains("probe_15"), - "body collapsed: {}", - out.text - ); - } + src.push_str("}\n"); - #[test] - fn per_body_tokens_retrieve_exactly_the_collapsed_body() { - let mut src = String::from("use std::io;\n\n"); - src.push_str("pub fn alpha() -> i32 {\n"); - for i in 0..10 { - src.push_str(&format!(" let a_{i} = compute({i});\n")); - } - src.push_str(" a_0\n}\n\n"); - src.push_str("pub fn beta() -> i32 {\n"); - for i in 0..10 { - src.push_str(&format!(" let b_{i} = compute({i});\n")); - } - src.push_str(" b_0\n}\n"); - - let out = compress_heuristic(&src, true, None, None).expect("compresses"); - let tokens = cache::parse_markers(&out.text); - assert_eq!( - tokens.len(), - 2, - "one token per collapsed body: {}", - out.text - ); - assert!( - out.text.contains("individually retrievable"), - "{}", - out.text - ); + let out = stub_code(&src, Some("rs"), &StubMode::SignaturesOnly, usize::MAX); - // Each token retrieves exactly its own body. - let first = cache::retrieve(&tokens[0]).expect("stored"); - assert!(first.contains("let a_5 = compute(5);"), "{first}"); - assert!(!first.contains("b_5"), "bodies must not mix: {first}"); - let second = cache::retrieve(&tokens[1]).expect("stored"); - assert!(second.contains("let b_5 = compute(5);"), "{second}"); + assert_eq!(out.parse_status, ParseStatus::TreeSitter); + assert!(out.text.contains("pub fn visible()")); + assert!(out.text.contains("lines")); + assert!(!out.text.contains("println!(\"10\")")); + assert_eq!(out.symbols[0].name, "visible"); + assert!(out.symbols[0].public); + assert_eq!(out.elisions.len(), 1); + assert!(out.elisions[0].start_line <= out.elisions[0].end_line); } #[test] - fn no_tokens_when_disabled() { - let mut src = String::from("use std::io;\n\n"); - src.push_str("pub fn gamma() -> i32 {\n"); - for i in 0..12 { - src.push_str(&format!(" let g_{i} = compute({i});\n")); + fn code_stub_transform_requires_retained_ccr() { + let mut src = String::from("use std::fmt;\n\npub fn visible() {\n"); + for i in 0..20 { + src.push_str(&format!(" println!(\"{i}\");\n")); } - src.push_str(" g_0\n}\n"); - let out = compress_heuristic(&src, false, None, None).expect("compresses"); - assert!(cache::parse_markers(&out.text).is_empty(), "{}", out.text); - assert!(!out.text.contains("individually retrievable")); + src.push_str("}\n"); + let pipeline_input = PipelineInput { + content: &src, + original_content: &src, + content_kind: ContentKind::Code, + original_bytes: src.len(), + }; + let transform = CodeStubTransform::new(StubMode::SignaturesOnly).with_extension("rs"); + + let rejecting_store = MemoryCcrStore::new(1, 1); + assert!(transform.apply(&pipeline_input, &rejecting_store).is_none()); + + let store = MemoryCcrStore::default(); + let out = transform + .apply(&pipeline_input, &store) + .expect("retained code stub"); + + assert_eq!(out.kind(), CompressorKind::Code); + assert!(out.text().contains("pub fn visible()"), "{}", out.text()); + assert!(!out.text().contains("println!(\"10\")"), "{}", out.text()); + assert_eq!(store.get(out.token()).as_deref(), Some(src.as_str())); } - #[cfg(feature = "tinyjuice-treesitter")] #[test] - fn treesitter_per_body_tokens_round_trip() { - let mut src = String::from("pub fn delta(items: &[i32]) -> i32 {\n"); - for i in 0..12 { - src.push_str(&format!( - " let d_{i} = items.iter().sum::() + {i};\n" - )); - } - src.push_str(" d_0\n}\n"); - let out = treesitter::compress(&src, "rs", true, None, None).expect("compresses"); - let tokens = cache::parse_markers(&out.text); - assert_eq!(tokens.len(), 1, "{}", out.text); - let body = cache::retrieve(&tokens[0]).expect("stored"); - assert!(body.contains("let d_7"), "{body}"); + fn code_stub_transform_skips_non_code_input() { + let input = PipelineInput { + content: "plain text", + original_content: "plain text", + content_kind: ContentKind::PlainText, + original_bytes: "plain text".len(), + }; + let transform = CodeStubTransform::new(StubMode::SignaturesOnly); + let store = MemoryCcrStore::default(); + + assert_eq!(transform.estimate_bloat(&input), 0.0); + assert!(transform.apply(&input, &store).is_none()); } - /// A body whose name or contents matches the caller's query stays open. #[test] - fn query_keeps_matching_function() { - let mut src = String::from("pub fn unrelated() -> i32 {\n"); - for i in 0..20 { - src.push_str(&format!(" let u_{i} = {i};\n")); + fn stub_code_expands_matched_symbol() { + let mut src = String::from("pub fn visible() {\n"); + for i in 0..10 { + src.push_str(&format!(" println!(\"visible {i}\");\n")); } - src.push_str(" u_0\n}\n\n"); - src.push_str("pub fn compute_checksum(data: &[u8]) -> u32 {\n"); - for i in 0..20 { - src.push_str(&format!(" let c_{i} = data.len() as u32 + {i};\n")); + src.push_str("}\n\nfn hidden() {\n"); + for i in 0..10 { + src.push_str(&format!(" println!(\"hidden {i}\");\n")); } - src.push_str(" c_0\n}\n"); - let out = compress_heuristic(&src, false, Some("checksum"), None).expect("compresses"); - // The queried function stays open; the unrelated one collapses. - assert!( - out.text.contains("let c_10"), - "queried body kept: {}", - out.text - ); - assert!( - !out.text.contains("u_10"), - "other body collapsed: {}", - out.text + src.push_str("}\n"); + + let out = stub_code( + &src, + Some("rs"), + &StubMode::MatchedSymbols(vec!["visible".to_string()]), + usize::MAX, ); + + assert!(out.text.contains("visible 9"), "{}", out.text); + assert!(!out.text.contains("hidden 9"), "{}", out.text); + assert_eq!(out.elisions.len(), 1); } - /// An error-handling dense body is kept even though it is long. #[test] - fn error_dense_body_is_kept() { - let mut src = String::from("pub fn validate(input: &str) -> Result<(), E> {\n"); - src.push_str(" if input.is_empty() {\n"); - src.push_str(" return Err(E::Empty);\n"); - src.push_str(" }\n"); - for i in 0..8 { - src.push_str(&format!(" let step_{i} = check(input, {i});\n")); + fn stub_code_public_api_omits_private_declarations() { + let mut src = String::from("use std::fmt;\n\npub fn visible() {\n"); + for i in 0..10 { + src.push_str(&format!(" println!(\"visible {i}\");\n")); } - src.push_str(" if bad {\n"); - src.push_str(" return Err(E::Bad);\n"); - src.push_str(" }\n"); - src.push_str(" Ok(())\n}\n\n"); - // Add an unrelated collapsible function so the file still shrinks. - src.push_str("pub fn filler() -> i32 {\n"); - for i in 0..20 { - src.push_str(&format!(" let f_{i} = {i};\n")); + src.push_str("}\n\nfn hidden() {\n"); + for i in 0..10 { + src.push_str(&format!(" println!(\"hidden {i}\");\n")); } - src.push_str(" f_0\n}\n"); - let out = compress_heuristic(&src, false, None, None).expect("compresses"); + src.push_str("}\n"); + + let out = stub_code(&src, Some("rs"), &StubMode::PublicApi, usize::MAX); + + assert!(out.text.contains("use std::fmt;"), "{}", out.text); + assert!(out.text.contains("pub fn visible()"), "{}", out.text); + assert!(out.text.contains("private declaration"), "{}", out.text); + assert!(!out.text.contains("fn hidden"), "{}", out.text); + assert!(!out.text.contains("hidden 9"), "{}", out.text); + assert_eq!(out.symbols.len(), 1); + assert_eq!(out.symbols[0].name, "visible"); + assert!(out.symbols[0].public); assert!( - out.text.contains("return Err(E::Bad)"), - "error-dense body kept: {}", - out.text + out.elisions + .iter() + .any(|elision| elision.reason == "public_api_private_declaration"), + "{:?}", + out.elisions ); } - /// The byte guard prevents collapsing a body when the marker would be - /// longer than the code it replaces (no inflation). #[test] - fn byte_guard_does_not_inflate() { - // A container of short one-line methods: each body is tiny, so even - // though there are many of them, none should be replaced by a marker - // that is longer than the body. - let mut src = String::from("class Tiny {\n"); - for i in 0..14 { - src.push_str(&format!(" fn m{i}(&self) -> i32 {{ {i} }}\n")); + fn stub_code_reports_heuristic_fallback_for_unknown_language() { + let mut src = String::from("function f() {\n"); + for i in 0..12 { + src.push_str(&format!(" call({i});\n")); } src.push_str("}\n"); - // Nothing collapses, so it does not shrink -> None (no inflation, no - // bogus compression). - let out = compress_heuristic(&src, false, None, None); - if let Some(o) = out { - assert!(o.text.len() < src.len(), "must never inflate: {}", o.text); - // The one-line method bodies must not have been swapped for a - // longer marker. - assert!( - !o.text.contains("{ … "), - "tiny bodies untouched: {}", - o.text - ); - } - } - /// PHP class: method signatures and PHPDoc survive, only long bodies - /// collapse. The class must never shrink to a single marker. - #[test] - fn php_class_keeps_method_signatures() { - let mut src = String::from("conn = $conn;\n }\n\n"); - src.push_str(" /**\n * Find a record by id.\n */\n"); - src.push_str(" public function findById(int $id): ?Record\n {\n"); - for i in 0..20 { - src.push_str(&format!(" $step_{i} = $this->query($id, {i});\n")); - } - src.push_str(" return $step_0;\n }\n\n"); - src.push_str(" public function deleteAll(): void\n {\n"); - for i in 0..20 { - src.push_str(&format!(" $this->purge({i});\n")); - } - src.push_str(" }\n}\n"); - let out = compress_heuristic(&src, false, None, None).expect("compresses"); - assert!(out.text.contains("class Repository"), "{}", out.text); - assert!( - out.text - .contains("public function findById(int $id): ?Record"), - "method signature kept: {}", - out.text - ); - assert!( - out.text.contains("public function deleteAll(): void"), - "method signature kept: {}", - out.text - ); - assert!( - out.text.contains("private ?Connection $conn;"), - "field kept: {}", - out.text - ); - assert!( - out.text.contains("Find a record by id"), - "phpdoc kept: {}", - out.text - ); - // Long bodies collapsed. - assert!(!out.text.contains("step_10"), "{}", out.text); - // Class must not have been reduced to a lone marker. - assert!( - out.text.matches("public function").count() >= 2, - "container must keep its member signatures: {}", - out.text - ); - } + let out = stub_code(&src, Some("unknown"), &StubMode::SignaturesOnly, usize::MAX); - /// The short constructor stays verbatim. - #[test] - fn php_class_keeps_short_constructor() { - let mut src = String::from("x = $x;\n $this->y = $y;\n }\n\n"); - src.push_str(" public function distance(Point $o): float\n {\n"); - for i in 0..20 { - src.push_str(&format!(" $d_{i} = hypot($this->x, {i});\n")); - } - src.push_str(" return $d_0;\n }\n}\n"); - let out = compress_heuristic(&src, false, None, None).expect("compresses"); - assert!( - out.text.contains("$this->x = $x;"), - "constructor body kept: {}", - out.text - ); - assert!( - !out.text.contains("$d_10"), - "long body collapsed: {}", - out.text - ); + assert_eq!(out.parse_status, ParseStatus::HeuristicFallback); + assert!(!out.elisions.is_empty()); + assert!(out.text.contains("lines")); } - /// Java class with Javadoc: class + method signatures + Javadoc survive. #[test] - fn java_class_with_javadoc_keeps_signatures() { - let mut src = String::from("package com.example;\n\n"); - src.push_str("/**\n * A widget registry. Uses {@link Widget}.\n */\n"); - src.push_str("public final class Registry {\n"); - src.push_str(" private final Map items = new HashMap<>();\n\n"); - src.push_str(" /**\n * Register a widget under a name.\n */\n"); - src.push_str(" public void register(String name, Widget w) {\n"); - for i in 0..20 { - src.push_str(&format!(" this.items.put(name + {i}, w);\n")); + fn stub_code_public_api_heuristic_omits_private_blocks() { + let mut src = String::from("export function visible() {\n"); + for i in 0..10 { + src.push_str(&format!(" console.log(\"visible {i}\");\n")); } - src.push_str(" }\n\n"); - src.push_str(" public Widget lookup(String name) {\n"); - for i in 0..20 { - src.push_str(&format!(" var candidate_{i} = items.get(name);\n")); + src.push_str("}\n\nfunction hidden() {\n"); + for i in 0..10 { + src.push_str(&format!(" console.log(\"hidden {i}\");\n")); } - src.push_str(" return null;\n }\n}\n"); - let out = compress_heuristic(&src, false, None, None).expect("compresses"); + src.push_str("}\n"); + + let out = stub_code(&src, Some("unknown"), &StubMode::PublicApi, usize::MAX); + + assert_eq!(out.parse_status, ParseStatus::HeuristicFallback); assert!( - out.text.contains("public final class Registry"), + out.text.contains("export function visible()"), "{}", out.text ); - assert!( - out.text - .contains("public void register(String name, Widget w)"), - "method signature kept: {}", - out.text - ); - assert!( - out.text.contains("public Widget lookup(String name)"), - "method signature kept: {}", - out.text - ); - assert!( - out.text.contains("Register a widget under a name"), - "javadoc kept: {}", - out.text - ); - assert!( - out.text.contains("private final Map items"), - "field kept: {}", - out.text - ); - assert!( - !out.text.contains("candidate_10"), - "body collapsed: {}", - out.text - ); + assert!(out.text.contains("private declaration"), "{}", out.text); + assert!(!out.text.contains("function hidden"), "{}", out.text); + assert!(!out.text.contains("hidden 9"), "{}", out.text); + assert_eq!(out.symbols.len(), 1); + assert_eq!(out.symbols[0].name, "visible"); + assert!(out.symbols[0].public); } - /// C# namespace + class: members inside the nested container stay visible - /// even though the namespace adds a level of nesting. #[test] - fn csharp_namespace_class_keeps_members() { - let mut src = String::from("using System;\n\nnamespace Acme.Core\n{\n"); - src.push_str(" public class Service\n {\n"); - src.push_str(" private readonly ILogger _log;\n\n"); - src.push_str(" public void Run(int count)\n {\n"); - for i in 0..20 { - src.push_str(&format!(" _log.Info($\"tick {i}\");\n")); + fn stub_code_matched_symbol_heuristic_expands_requested_body() { + let mut src = String::from("function visible() {\n"); + for i in 0..10 { + src.push_str(&format!(" console.log(\"visible {i}\");\n")); } - src.push_str(" }\n\n"); - src.push_str(" public int Compute(int seed)\n {\n"); - for i in 0..20 { - src.push_str(&format!(" var v_{i} = seed + {i};\n")); + src.push_str("}\n\nfunction hidden() {\n"); + for i in 0..10 { + src.push_str(&format!(" console.log(\"hidden {i}\");\n")); } - src.push_str(" return seed;\n }\n }\n}\n"); - let out = compress_heuristic(&src, false, None, None).expect("compresses"); - assert!(out.text.contains("public class Service"), "{}", out.text); - assert!( - out.text.contains("public void Run(int count)"), - "method signature kept: {}", - out.text - ); - assert!( - out.text.contains("public int Compute(int seed)"), - "method signature kept: {}", - out.text - ); - assert!( - out.text.contains("private readonly ILogger _log;"), - "field kept: {}", - out.text - ); - assert!(!out.text.contains("v_10"), "body collapsed: {}", out.text); - } + src.push_str("}\n"); - /// C++ class inside a namespace keeps its method signatures. - #[test] - fn cpp_namespace_class_keeps_signatures() { - let mut src = String::from("#include \n\nnamespace engine {\n\n"); - src.push_str("class SceneNode {\npublic:\n"); - src.push_str(" void attach(SceneNode* child) {\n"); - for i in 0..20 { - src.push_str(&format!(" this->children.push_back({i});\n")); - } - src.push_str(" }\n"); - src.push_str(" void detach(SceneNode* child) {\n"); - for i in 0..20 { - src.push_str(&format!(" this->children.erase({i});\n")); - } - src.push_str(" }\n"); - src.push_str("};\n\n} // namespace engine\n"); - let out = compress_heuristic(&src, false, None, None).expect("compresses"); - assert!( - out.text.contains("class SceneNode"), - "class inside namespace must stay visible: {}", - out.text - ); - assert!( - out.text.contains("void attach(SceneNode* child)"), - "method signature kept: {}", - out.text + let out = stub_code( + &src, + Some("unknown"), + &StubMode::MatchedSymbols(vec!["visible".to_string()]), + usize::MAX, ); - assert!( - out.text.contains("void detach(SceneNode* child)"), - "method signature kept: {}", - out.text - ); - } - #[test] - fn block_comments_do_not_drift_depth() { - // Javadoc braces ({@link ...}) must not count. - let mut in_c = false; - assert_eq!( - brace_delta("/** see {@link Foo#bar} for details", &mut in_c), - (0, 0) - ); - assert!(in_c); - assert_eq!( - brace_delta(" * more prose with { braces } inside", &mut in_c), - (0, 0) + assert_eq!(out.parse_status, ParseStatus::HeuristicFallback); + assert!(out.text.contains("visible 9"), "{}", out.text); + assert!(!out.text.contains("hidden 9"), "{}", out.text); + assert_eq!(out.symbols.len(), 2); + assert!( + out.elisions + .iter() + .any(|elision| elision.reason.contains("MatchedSymbols")), + "{:?}", + out.elisions ); - assert_eq!(brace_delta(" */ fn real() {", &mut in_c), (1, 0)); - assert!(!in_c); } #[test] - fn fn_name_extracts_across_languages() { - assert_eq!( - fn_name("pub fn process(items: &[i32]) -> i32 {").as_deref(), - Some("process") - ); - assert_eq!(fn_name("def handler(event):").as_deref(), Some("handler")); - assert_eq!( - fn_name("public function getRoot(): ?AVLTreeNode").as_deref(), - Some("getRoot") - ); - assert_eq!( - fn_name("public void doThing(int x) {").as_deref(), - Some("doThing") - ); - assert_eq!( - fn_name("func (r *Repo) Save(x int) error {").as_deref(), - Some("Save") - ); - assert_eq!( - fn_name("func Serve(addr string) {").as_deref(), - Some("Serve") - ); - } - - /// Build a file with one huge body plus several small (but still eligible) - /// bodies, so budget selection has a clear largest-first choice. - fn huge_plus_small_bodies() -> String { - let mut src = String::from("use std::io;\n\n"); - src.push_str("pub fn giant(items: &[i32]) -> i32 {\n"); - for i in 0..80 { - src.push_str(&format!( - " let giant_{i} = items.iter().sum::() + {i};\n" - )); - } - src.push_str(" giant_0\n}\n"); - for f in 0..3 { - src.push_str(&format!("\npub fn small_{f}(x: i32) -> i32 {{\n")); - for i in 0..9 { - src.push_str(&format!(" let small_{f}_{i} = x + {i};\n")); - } - src.push_str(&format!(" small_{f}_0\n}}\n")); + fn stub_code_expand_around_lines_heuristic_expands_containing_body() { + let mut src = String::from("function first() {\n"); + for i in 0..10 { + src.push_str(&format!(" console.log(\"first {i}\");\n")); } - src - } - - /// With a lenient target ratio only the largest body collapses; the small - /// eligible bodies stay fully intact. - #[test] - fn target_ratio_collapses_only_largest_body() { - let src = huge_plus_small_bodies(); - let out = compress_heuristic(&src, false, None, Some(0.9)).expect("compresses"); - assert!( - !out.text.contains("giant_40"), - "largest body collapsed: {}", - out.text - ); - for f in 0..3 { - assert!( - out.text.contains(&format!("let small_{f}_5 = x + 5;")), - "small body {f} left intact: {}", - out.text - ); + src.push_str("}\n\nfunction second() {\n"); + for i in 0..10 { + src.push_str(&format!(" console.log(\"second {i}\");\n")); } - assert!( - (out.text.len() as f32) <= 0.9 * src.len() as f32, - "target met: {} vs {}", - out.text.len(), - src.len() - ); - } - - /// A very small target ratio collapses every eligible body — identical to - /// the default collapse-everything behavior. - #[test] - fn tiny_target_ratio_matches_collapse_all() { - let src = huge_plus_small_bodies(); - let all = compress_heuristic(&src, false, None, None).expect("compresses"); - let budgeted = compress_heuristic(&src, false, None, Some(0.01)).expect("compresses"); - assert_eq!(all.text, budgeted.text); - assert!(!budgeted.text.contains("small_1_5"), "{}", budgeted.text); - } + src.push_str("}\n"); - /// Projected-size accounting: with several equal bodies and an achievable - /// mid-range target, the output lands at or under the target while some - /// bodies are left uncollapsed (early stop). - #[test] - fn target_ratio_projection_is_sane() { - let mut src = String::from("use std::io;\n\n"); - for f in 0..6 { - src.push_str(&format!("pub fn worker_{f}(x: i32) -> i32 {{\n")); - for i in 0..20 { - src.push_str(&format!(" let step_{f}_{i} = compute(x, {i});\n")); - } - src.push_str(&format!(" step_{f}_0\n}}\n\n")); - } - let full = compress_heuristic(&src, false, None, None).expect("compresses"); - let out = compress_heuristic(&src, false, None, Some(0.6)).expect("compresses"); - assert!( - (out.text.len() as f32) <= 0.6 * src.len() as f32, - "output within target: {} vs input {}", - out.text.len(), - src.len() - ); - assert!( - out.text.len() > full.text.len(), - "early stop leaves some bodies open: {} vs fully collapsed {}", - out.text.len(), - full.text.len() + let out = stub_code( + &src, + Some("unknown"), + &StubMode::ExpandAroundLines(vec![LineRange::new(7, 7)]), + usize::MAX, ); - let open = (0..6) - .filter(|f| out.text.contains(&format!("step_{f}_10"))) - .count(); - assert!( - (1..=5).contains(&open), - "some but not all bodies stay open (open={open}): {}", - out.text - ); - } - /// Tree-sitter path honors the target ratio the same way: largest first, - /// stop once the budget is met. - #[cfg(feature = "tinyjuice-treesitter")] - #[test] - fn treesitter_target_ratio_collapses_only_largest() { - let src = huge_plus_small_bodies(); - let out = treesitter::compress(&src, "rs", false, None, Some(0.9)).expect("compresses"); + assert_eq!(out.parse_status, ParseStatus::HeuristicFallback); + assert!(out.text.contains("first 9"), "{}", out.text); + assert!(!out.text.contains("second 9"), "{}", out.text); + assert_eq!(out.symbols.len(), 2); assert!( - !out.text.contains("giant_40"), - "largest body collapsed: {}", - out.text + out.elisions + .iter() + .any(|elision| elision.reason.contains("ExpandAroundLines")), + "{:?}", + out.elisions ); - for f in 0..3 { - assert!( - out.text.contains(&format!("let small_{f}_5 = x + 5;")), - "small body {f} left intact: {}", - out.text - ); - } - // And a tiny ratio matches collapse-all. - let all = treesitter::compress(&src, "rs", false, None, None).expect("compresses"); - let budgeted = - treesitter::compress(&src, "rs", false, None, Some(0.01)).expect("compresses"); - assert_eq!(all.text, budgeted.text); - } - - #[test] - fn decl_opener_and_signature_classification() { - assert!(is_decl_opener_line("public void doThing(int x) {")); - assert!(is_decl_opener_line("class Foo {")); - assert!(is_decl_opener_line("pub fn f<'a>(x: &'a str) {")); - assert!(!is_decl_opener_line("if (x > 0) {")); - assert!(!is_decl_opener_line("for (int i = 0; i < n; i++) {")); - assert!(!is_decl_opener_line("{")); - assert!(is_signature_line("public function getRoot(): ?AVLTreeNode")); - assert!(!is_signature_line("$this->root = null;")); - assert!(!is_signature_line("return doThing(x);")); } } diff --git a/src/compressors/diff.rs b/src/compressors/diff.rs index 342d9ba..b60d5ce 100644 --- a/src/compressors/diff.rs +++ b/src/compressors/diff.rs @@ -9,17 +9,102 @@ use async_trait::async_trait; use std::fmt::Write as _; -use super::anchors::extract_anchors; -use super::{BLOCK_NOTE, Compressor, block_token}; +use super::Compressor; +use crate::cache::CcrStore; +use crate::pipeline::{OffloadOutput, OffloadTransform, PipelineInput, estimate_bloat}; use crate::text::ansi::strip_ansi; -use crate::types::{CompressInput, CompressOptions, CompressOutput, CompressorKind}; +use crate::types::{CompressInput, CompressOptions, CompressOutput, CompressorKind, ContentKind}; /// Context lines kept on each side of a changed run before collapsing. pub const CONTEXT_ANCHOR: usize = 3; /// A run of unchanged context longer than this collapses to a marker. pub const CONTEXT_COLLAPSE_THRESHOLD: usize = 8; -/// Query words shorter than this are ignored as anchors (too noisy). -pub const MIN_QUERY_WORD: usize = 3; + +const DEFAULT_LOCKFILE_PATH_PATTERNS: &[&str] = &[ + "cargo.lock", + "package-lock.json", + "pnpm-lock.yaml", + "yarn.lock", + "composer.lock", + "poetry.lock", + "gemfile.lock", + "go.sum", +]; +const DEFAULT_GENERATED_BUNDLE_PATH_PATTERNS: &[&str] = &[".min.js", ".min.css", ".map"]; + +/// Controls the low-value diff hunk offload policy. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct DiffNoiseOptions { + /// Case-insensitive path substrings that identify generated or lockfile + /// hunks whose body can be represented by a reason marker. + pub noisy_path_substrings: Vec, + /// Drop hunks whose added/removed lines are identical after ASCII + /// whitespace stripping. Off by default until fixture coverage is broad. + pub drop_whitespace_only_hunks: bool, +} + +impl Default for DiffNoiseOptions { + fn default() -> Self { + Self { + noisy_path_substrings: DEFAULT_LOCKFILE_PATH_PATTERNS + .iter() + .map(|value| value.to_string()) + .chain( + DEFAULT_GENERATED_BUNDLE_PATH_PATTERNS + .iter() + .map(|value| value.to_string()), + ) + .collect(), + drop_whitespace_only_hunks: false, + } + } +} + +/// Typed offload transform for low-value diff hunks. +pub struct DiffNoiseTransform { + options: DiffNoiseOptions, +} + +impl DiffNoiseTransform { + pub fn new(options: DiffNoiseOptions) -> Self { + Self { options } + } + + pub fn options(&self) -> &DiffNoiseOptions { + &self.options + } +} + +impl Default for DiffNoiseTransform { + fn default() -> Self { + Self::new(DiffNoiseOptions::default()) + } +} + +impl OffloadTransform for DiffNoiseTransform { + fn name(&self) -> &'static str { + "diff_noise" + } + + fn estimate_bloat(&self, input: &PipelineInput<'_>) -> f32 { + if input.content_kind != ContentKind::Diff { + return 0.0; + } + f32::from(estimate_bloat(input.content, input.content_kind).score) / 100.0 + } + + fn apply(&self, input: &PipelineInput<'_>, store: &dyn CcrStore) -> Option { + if input.content_kind != ContentKind::Diff { + return None; + } + let compacted = compress_with_options(input.content, &self.options)?; + OffloadOutput::from_retained_put( + compacted.text, + CompressorKind::Diff, + store.put(input.original_content), + ) + } +} pub struct DiffCompressor; @@ -32,37 +117,39 @@ impl Compressor for DiffCompressor { async fn compress( &self, input: &CompressInput<'_>, - opts: &CompressOptions, + _opts: &CompressOptions, ) -> Option { - compress(input.content, opts.ccr_enabled, input.hint.query.as_deref()) + compress(input.content) } } -/// Compress a unified diff. With `block_tokens`, each omitted block (collapsed -/// context run, summarized lockfile hunk, whitespace-only hunk) is offloaded to -/// CCR and its marker carries a token retrieving exactly that block. `query` -/// yields anchors/words that, when they appear in a context run, prevent that -/// run from being collapsed. Returns `None` when there's nothing structural to +/// Compress a unified diff. Returns `None` when there's nothing structural to /// work with or compression wouldn't shrink it. -pub fn compress(content: &str, block_tokens: bool, query: Option<&str>) -> Option { +pub fn compress(content: &str) -> Option { + compress_with_options(content, &DiffNoiseOptions::default()) +} + +/// Compress a unified diff with explicit diff-noise policy. +pub fn compress_with_options( + content: &str, + noise_options: &DiffNoiseOptions, +) -> Option { let stripped = strip_ansi(content); let lines: Vec<&str> = stripped.lines().collect(); if lines.is_empty() { return None; } - let keepers = query_keepers(query); let mut out = String::with_capacity(stripped.len() / 2 + 64); let mut i = 0usize; - let mut current_file_is_noisy = false; + let mut current_file_noise_reason = None; let mut saw_hunk = false; - let mut any_token = false; while i < lines.len() { let line = lines[i]; if line.starts_with("diff --git ") { - current_file_is_noisy = is_noisy_path(line); + current_file_noise_reason = diff_noise_reason(line, noise_options); let _ = writeln!(out, "{line}"); i += 1; continue; @@ -71,38 +158,29 @@ pub fn compress(content: &str, block_tokens: bool, query: Option<&str>) -> Optio saw_hunk |= line.starts_with("@@"); if line.starts_with("@@") { let _ = writeln!(out, "{line}"); - i += 1; - let (added, removed, consumed) = summarize_hunk_body(&lines[i..]); - let body = &lines[i..i + consumed]; - let has_keeper = lines_match_keeper(body, &keepers); - - if current_file_is_noisy { - let token = block_token(&body.join("\n"), block_tokens); - any_token |= !token.is_empty(); - let _ = writeln!( - out, - "[... lockfile/bundle hunk: +{added}/-{removed} line(s) omitted ...{token}]" - ); - i += consumed; - continue; - } - - // Whitespace-only hunk: every -/+ pair is identical once all - // whitespace is collapsed → semantically a no-op. Collapse it - // like a lockfile hunk, unless a query anchor mentions it. - if !has_keeper && is_whitespace_noop_hunk(body) { - let token = block_token(&body.join("\n"), block_tokens); - any_token |= !token.is_empty(); + let hunk_start = i + 1; + let hunk_end = hunk_body_end(&lines, hunk_start); + let hunk_body = &lines[hunk_start..hunk_end]; + let reason = if let Some(reason) = current_file_noise_reason { + Some(reason) + } else if noise_options.drop_whitespace_only_hunks + && hunk_is_whitespace_only(hunk_body) + { + Some("whitespace_only") + } else { + None + }; + if let Some(reason) = reason { + let summary = summarize_hunk_body(hunk_body); let _ = writeln!( out, - "[... whitespace-only hunk: {consumed} line(s) omitted (formatting-only) ...{token}]" + "[... diff-noise hunk omitted reason={reason} +{}/-{} context={} line(s) ...]", + summary.added, summary.removed, summary.context ); - i += consumed; + i = hunk_end; continue; } - - // Otherwise fall through: let the main loop process the body - // (changed lines kept, long context runs collapsed). + i += 1; continue; } let _ = writeln!(out, "{line}"); @@ -116,18 +194,12 @@ pub fn compress(content: &str, block_tokens: bool, query: Option<&str>) -> Optio i += 1; } let run = &lines[start..i]; - // A query anchor/word inside the run pins it — never collapse. - if run.len() > CONTEXT_COLLAPSE_THRESHOLD && !lines_match_keeper(run, &keepers) { + if run.len() > CONTEXT_COLLAPSE_THRESHOLD { for l in &run[..CONTEXT_ANCHOR] { let _ = writeln!(out, "{l}"); } let omitted = run.len() - 2 * CONTEXT_ANCHOR; - let token = block_token( - &run[CONTEXT_ANCHOR..run.len() - CONTEXT_ANCHOR].join("\n"), - block_tokens, - ); - any_token |= !token.is_empty(); - let _ = writeln!(out, "[... {omitted} context line(s) omitted ...{token}]"); + let _ = writeln!(out, "[... {omitted} context line(s) omitted ...]"); for l in &run[run.len() - CONTEXT_ANCHOR..] { let _ = writeln!(out, "{l}"); } @@ -146,15 +218,11 @@ pub fn compress(content: &str, block_tokens: bool, query: Option<&str>) -> Optio if !saw_hunk { return None; } - if any_token { - out.push_str(BLOCK_NOTE); - out.push('\n'); - } if out.len() >= content.len() { return None; } log::debug!( - "[tinyjuice][diff] {} -> {} bytes ({} input lines)", + "[tokenjuice][diff] {} -> {} bytes ({} input lines)", content.len(), out.len(), lines.len(), @@ -181,113 +249,92 @@ fn is_context(line: &str) -> bool { line.starts_with(' ') } -fn summarize_hunk_body(lines: &[&str]) -> (usize, usize, usize) { - let mut added = 0usize; - let mut removed = 0usize; - let mut n = 0usize; +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +struct HunkSummary { + added: usize, + removed: usize, + context: usize, +} + +fn hunk_body_end(lines: &[&str], start: usize) -> usize { + let mut i = start; + while i < lines.len() && !lines[i].starts_with("@@") && !lines[i].starts_with("diff --git ") { + i += 1; + } + i +} + +fn summarize_hunk_body(lines: &[&str]) -> HunkSummary { + let mut summary = HunkSummary::default(); for &line in lines { - if line.starts_with("@@") || line.starts_with("diff --git ") { - break; - } if line.starts_with('+') && !line.starts_with("+++") { - added += 1; + summary.added += 1; } else if line.starts_with('-') && !line.starts_with("---") { - removed += 1; + summary.removed += 1; + } else if line.starts_with(' ') { + summary.context += 1; } - n += 1; } - (added, removed, n) + summary } -/// Build the set of query-derived keeper tokens: exact-match anchors plus query -/// words longer than two chars. Lower-cased for case-insensitive matching. -fn query_keepers(query: Option<&str>) -> Vec { - let Some(q) = query else { - return Vec::new(); - }; - let mut set: std::collections::BTreeSet = extract_anchors(q).into_iter().collect(); - for w in q.split(|c: char| !c.is_alphanumeric() && c != '_') { - if w.chars().count() >= MIN_QUERY_WORD { - set.insert(w.to_ascii_lowercase()); - } - } - set.into_iter().collect() +fn hunk_is_whitespace_only(lines: &[&str]) -> bool { + let removed: Vec = lines + .iter() + .filter_map(|line| line.strip_prefix('-')) + .map(strip_ascii_whitespace) + .collect(); + let added: Vec = lines + .iter() + .filter_map(|line| line.strip_prefix('+')) + .map(strip_ascii_whitespace) + .collect(); + !removed.is_empty() && removed.len() == added.len() && removed == added } -/// True if any line (case-insensitively) contains any keeper token. -fn lines_match_keeper(lines: &[&str], keepers: &[String]) -> bool { - if keepers.is_empty() { - return false; - } - lines.iter().any(|line| { - let lower = line.to_ascii_lowercase(); - keepers.iter().any(|k| lower.contains(k)) - }) +fn strip_ascii_whitespace(s: &str) -> String { + s.chars().filter(|c| !c.is_ascii_whitespace()).collect() } -/// True if `body` is a whitespace-only (formatting) hunk: it has at least one -/// changed line, an equal number of removals and additions, and every removal -/// pairs (in order) with an addition that is identical once all whitespace is -/// collapsed. Such a hunk carries no semantic change. -fn is_whitespace_noop_hunk(body: &[&str]) -> bool { - let mut removes: Vec = Vec::new(); - let mut adds: Vec = Vec::new(); - for &line in body { - if line.starts_with("+++") || line.starts_with("---") { - return false; +fn diff_noise_reason(diff_git_line: &str, options: &DiffNoiseOptions) -> Option<&'static str> { + let l = diff_git_line.to_ascii_lowercase(); + let paths = diff_git_paths(diff_git_line); + for pattern in &options.noisy_path_substrings { + let pattern = pattern.to_ascii_lowercase(); + if pattern.is_empty() + || !(l.contains(&pattern) || paths.iter().any(|path| path.ends_with(&pattern))) + { + continue; } - if let Some(rest) = line.strip_prefix('+') { - adds.push(collapse_ws(rest)); - } else if let Some(rest) = line.strip_prefix('-') { - removes.push(collapse_ws(rest)); + if pattern_matches_any_default(&pattern, DEFAULT_LOCKFILE_PATH_PATTERNS) { + return Some("lockfile"); } - // Context / other lines don't affect the whitespace-only decision. - } - if removes.is_empty() || removes.len() != adds.len() { - return false; + if pattern_matches_any_default(&pattern, DEFAULT_GENERATED_BUNDLE_PATH_PATTERNS) { + return Some("generated_bundle"); + } + return Some("configured_noisy_path"); } - removes.iter().zip(&adds).all(|(r, a)| r == a) + None } -/// Remove every whitespace character from `s` (used to compare lines modulo -/// indentation/spacing). -fn collapse_ws(s: &str) -> String { - s.chars().filter(|c| !c.is_whitespace()).collect() +fn pattern_matches_any_default(pattern: &str, defaults: &[&str]) -> bool { + defaults.contains(&pattern) } -fn is_noisy_path(diff_git_line: &str) -> bool { - // Match on the parsed file basenames, not the raw line: a substring - // check would flag real source files like `user.mapper.ts` (".map") or - // `heat.map.config.js` as bundle noise and drop their hunks. - const NOISY_BASENAMES: &[&str] = &[ - "cargo.lock", - "package-lock.json", - "pnpm-lock.yaml", - "yarn.lock", - "composer.lock", - "poetry.lock", - "gemfile.lock", - "go.sum", - ]; - const NOISY_SUFFIXES: &[&str] = &[".min.js", ".min.css", ".map"]; - - let Some(rest) = diff_git_line.strip_prefix("diff --git ") else { - return false; - }; - rest.split_whitespace().any(|tok| { - let path = tok - .strip_prefix("a/") - .or_else(|| tok.strip_prefix("b/")) - .unwrap_or(tok) - .to_ascii_lowercase(); - let basename = path.rsplit('/').next().unwrap_or(path.as_str()); - NOISY_BASENAMES.contains(&basename) || NOISY_SUFFIXES.iter().any(|s| basename.ends_with(s)) - }) +fn diff_git_paths(diff_git_line: &str) -> Vec { + diff_git_line + .split_whitespace() + .skip(2) + .filter_map(|path| path.strip_prefix("a/").or_else(|| path.strip_prefix("b/"))) + .map(|path| path.to_ascii_lowercase()) + .collect() } #[cfg(test)] mod tests { use super::*; + use crate::cache::{CcrStore, MemoryCcrStore}; + use crate::types::ContentKind; #[test] fn keeps_changed_lines_collapses_context() { @@ -300,7 +347,7 @@ mod tests { for i in 0..20 { let _ = writeln!(s, " more context {i} unchanged"); } - let out = compress(&s, false, None).expect("compresses").text; + let out = compress(&s).expect("compresses").text; assert!(out.contains("-old changed line"), "{out}"); assert!(out.contains("+new changed line"), "{out}"); assert!(out.contains("context line(s) omitted"), "{out}"); @@ -317,166 +364,204 @@ mod tests { for i in 0..20 { let _ = writeln!(s, "- old dep entry {i}"); } - let out = compress(&s, false, None).expect("compresses").text; - assert!(out.contains("lockfile/bundle hunk"), "{out}"); + let out = compress(&s).expect("compresses").text; + assert!( + out.contains("diff-noise hunk omitted reason=lockfile"), + "{out}" + ); assert!(!out.contains("new dep entry 7"), "{out}"); assert!(out.len() < s.len()); } #[test] - fn non_diff_returns_none() { - assert!(compress("just some text\nno hunks here", false, None).is_none()); - } - - #[test] - fn source_files_with_map_in_name_are_not_noisy() { - for line in [ - "diff --git a/src/user.mapper.ts b/src/user.mapper.ts", - "diff --git a/heat.map.config.js b/heat.map.config.js", - "diff --git a/src/routes/url.mapping.rs b/src/routes/url.mapping.rs", - "diff --git a/minutes.md b/minutes.md", - ] { - assert!(!is_noisy_path(line), "misclassified as noisy: {line}"); + fn empty_noisy_path_patterns_disable_lockfile_omission() { + let mut s = String::from("diff --git a/Cargo.lock b/Cargo.lock\n@@ -1,60 +1,80 @@\n"); + for i in 0..20 { + let _ = writeln!(s, " context before {i}"); + } + for i in 0..20 { + let _ = writeln!(s, "+ new dep entry {i}"); } + for i in 0..20 { + let _ = writeln!(s, "- old dep entry {i}"); + } + for i in 0..20 { + let _ = writeln!(s, " context after {i}"); + } + let options = DiffNoiseOptions { + noisy_path_substrings: Vec::new(), + ..Default::default() + }; + + let out = compress_with_options(&s, &options) + .expect("compresses") + .text; + + assert!(!out.contains("diff-noise hunk omitted"), "{out}"); + assert!(out.contains("new dep entry 7"), "{out}"); } #[test] - fn lockfiles_and_artifacts_are_noisy() { - for line in [ - "diff --git a/Cargo.lock b/Cargo.lock", - "diff --git a/vendor/pkg/go.sum b/vendor/pkg/go.sum", - "diff --git a/dist/app.min.js b/dist/app.min.js", - "diff --git a/dist/bundle.js.map b/dist/bundle.js.map", - "diff --git a/web/package-lock.json b/web/package-lock.json", - ] { - assert!(is_noisy_path(line), "should be noisy: {line}"); + fn summarizes_generated_bundle_hunk_with_reason() { + let mut s = + String::from("diff --git a/dist/app.min.js b/dist/app.min.js\n@@ -1,80 +1,80 @@\n"); + for i in 0..40 { + let _ = writeln!(s, "+ minified chunk payload {i}"); } + for i in 0..40 { + let _ = writeln!(s, "- previous minified payload {i}"); + } + + let out = compress(&s).expect("compresses").text; + + assert!( + out.contains("diff-noise hunk omitted reason=generated_bundle"), + "{out}" + ); + assert!(!out.contains("minified chunk payload 7"), "{out}"); } #[test] - fn source_file_hunks_survive_even_with_mapper_name() { + fn custom_noisy_path_patterns_get_configured_reason() { let mut s = String::from( - "diff --git a/src/user.mapper.ts b/src/user.mapper.ts\n@@ -1,40 +1,42 @@\n", + "diff --git a/fixtures/snapshot.txt b/fixtures/snapshot.txt\n@@ -1,80 +1,80 @@\n", ); - for i in 0..20 { - let _ = writeln!(s, " context line {i} unchanged here"); + for i in 0..40 { + let _ = writeln!(s, "+ snapshot chunk payload {i}"); } - let _ = writeln!(s, "-export function oldMap() {{}}"); - let _ = writeln!(s, "+export function newMap7() {{}}"); - for i in 0..20 { - let _ = writeln!(s, " more context {i} unchanged"); + for i in 0..40 { + let _ = writeln!(s, "- previous snapshot payload {i}"); } - let out = compress(&s, false, None).expect("compresses").text; - assert!(out.contains("+export function newMap7()"), "{out}"); - assert!(!out.contains("lockfile/bundle hunk"), "{out}"); + let options = DiffNoiseOptions { + noisy_path_substrings: vec!["snapshot.txt".to_string()], + ..Default::default() + }; + + let out = compress_with_options(&s, &options) + .expect("compresses") + .text; + + assert!( + out.contains("diff-noise hunk omitted reason=configured_noisy_path"), + "{out}" + ); + assert!(!out.contains("snapshot chunk payload 7"), "{out}"); } #[test] - fn context_block_token_retrieves_omitted_lines() { - use crate::cache; - let mut s = String::from("diff --git a/x.rs b/x.rs\n@@ -1,40 +1,41 @@\n"); + fn whitespace_only_hunks_are_preserved_by_default() { + let mut s = String::from("diff --git a/src/lib.rs b/src/lib.rs\n@@ -1,40 +1,40 @@\n"); for i in 0..20 { - let _ = writeln!(s, " context line {i} unchanged here"); + let _ = writeln!(s, " context before {i}"); } - let _ = writeln!(s, "-old changed line"); - let _ = writeln!(s, "+new changed line"); + let _ = writeln!(s, "-fn main(){{println!(\"hi\");}}"); + let _ = writeln!(s, "+fn main() {{ println!(\"hi\"); }}"); for i in 0..20 { - let _ = writeln!(s, " more context {i} unchanged"); + let _ = writeln!(s, " context after {i}"); } - let out = compress(&s, true, None).expect("compresses").text; - let tokens = cache::parse_markers(&out); - assert!( - !tokens.is_empty(), - "context marker should carry a token: {out}" - ); - let block = cache::retrieve(&tokens[0]).expect("stored"); - assert!(block.contains("context line 10 unchanged here"), "{block}"); - assert!(out.contains("individually retrievable"), "{out}"); + let out = compress(&s).expect("compresses").text; + + assert!(out.contains("-fn main()"), "{out}"); + assert!(out.contains("+fn main()"), "{out}"); + assert!(!out.contains("reason=whitespace_only"), "{out}"); } #[test] - fn whitespace_only_hunk_collapsed_and_recoverable() { - use crate::cache; - // Feature 9: every -/+ pair differs only in leading indentation → the - // hunk is a formatting no-op and collapses to a one-line note, with the - // full body recoverable via the per-block token. - let mut s = String::from("diff --git a/src/x.rs b/src/x.rs\n@@ -1,40 +1,40 @@\n"); - for i in 0..20 { - let _ = writeln!(s, "- let value_{i} = compute();"); - let _ = writeln!(s, "+ let value_{i} = compute();"); + fn whitespace_only_hunks_drop_when_enabled() { + let mut s = String::from("diff --git a/src/lib.rs b/src/lib.rs\n@@ -1,60 +1,60 @@\n"); + for i in 0..50 { + let _ = writeln!(s, " context before {i}"); } - let out = compress(&s, true, None).expect("compresses").text; - assert!(out.contains("whitespace-only hunk"), "{out}"); - assert!( - !out.contains("let value_5"), - "body should be collapsed:\n{out}" - ); - assert!(out.len() < s.len()); + let _ = writeln!(s, "-fn main(){{println!(\"hi\");}}"); + let _ = writeln!(s, "+fn main() {{ println!(\"hi\"); }}"); + for i in 0..50 { + let _ = writeln!(s, " context after {i}"); + } + let options = DiffNoiseOptions { + drop_whitespace_only_hunks: true, + ..Default::default() + }; + let out = compress_with_options(&s, &options) + .expect("compresses") + .text; - let tokens = cache::parse_markers(&out); assert!( - !tokens.is_empty(), - "whitespace hunk should carry a token: {out}" + out.contains("diff-noise hunk omitted reason=whitespace_only"), + "{out}" ); - let block = cache::retrieve(&tokens[0]).expect("stored"); - assert!(block.contains("let value_5 = compute();"), "{block}"); + assert!(!out.contains("-fn main()"), "{out}"); + assert!(!out.contains("+fn main()"), "{out}"); } #[test] - fn real_change_hunk_not_treated_as_whitespace_noop() { - // A hunk that genuinely changes a token must NOT collapse as whitespace. - let mut s = String::from("diff --git a/src/x.rs b/src/x.rs\n@@ -1,3 +1,3 @@\n"); - let _ = writeln!(s, "-let total = old_value + 1;"); - let _ = writeln!(s, "+let total = new_value + 1;"); + fn semantic_hunks_survive_whitespace_noise_mode() { + let mut s = String::from("diff --git a/src/lib.rs b/src/lib.rs\n@@ -1,40 +1,40 @@\n"); + for i in 0..20 { + let _ = writeln!(s, " context before {i}"); + } + let _ = writeln!(s, "-fn answer() -> i32 {{ 41 }}"); + let _ = writeln!(s, "+fn answer() -> i32 {{ 42 }}"); for i in 0..20 { - let _ = writeln!(s, " context tail {i}"); + let _ = writeln!(s, " context after {i}"); } - let out = compress(&s, false, None).expect("compresses").text; - assert!(!out.contains("whitespace-only hunk"), "{out}"); - assert!(out.contains("+let total = new_value + 1;"), "{out}"); + let options = DiffNoiseOptions { + drop_whitespace_only_hunks: true, + ..Default::default() + }; + let out = compress_with_options(&s, &options) + .expect("compresses") + .text; + + assert!(out.contains("41"), "{out}"); + assert!(out.contains("42"), "{out}"); + assert!(!out.contains("diff-noise hunk omitted"), "{out}"); } #[test] - fn query_word_prevents_context_collapse() { - // Feature 10: a long context run mentioning a query word is pinned; a - // second file's plain context run still collapses so the output shrinks. - let mut s = String::new(); - let _ = writeln!(s, "diff --git a/auth.rs b/auth.rs"); - let _ = writeln!(s, "@@ -1,25 +1,26 @@"); - for i in 0..20 { - if i == 10 { - let _ = writeln!(s, " let token = AUTHENTICATION_TOKEN;"); - } else { - let _ = writeln!(s, " auth context {i}"); - } + fn diff_noise_transform_requires_retained_ccr() { + let mut s = String::from("diff --git a/Cargo.lock b/Cargo.lock\n@@ -1,80 +1,80 @@\n"); + for i in 0..80 { + let _ = writeln!(s, "+ new dep entry {i}"); } - let _ = writeln!(s, "-old auth line"); - let _ = writeln!(s, "+new auth line"); - let _ = writeln!(s, "diff --git a/other.rs b/other.rs"); - let _ = writeln!(s, "@@ -1,22 +1,23 @@"); - for i in 0..20 { - let _ = writeln!(s, " other context {i}"); + for i in 0..80 { + let _ = writeln!(s, "- old dep entry {i}"); } - let _ = writeln!(s, "-old other line"); - let _ = writeln!(s, "+new other line"); + let input = PipelineInput { + content: &s, + original_content: &s, + content_kind: ContentKind::Diff, + original_bytes: s.len(), + }; + let transform = DiffNoiseTransform::default(); + + let rejecting_store = MemoryCcrStore::new(1, 1); + assert!(transform.apply(&input, &rejecting_store).is_none()); + + let store = MemoryCcrStore::default(); + let out = transform.apply(&input, &store).expect("retained offload"); + + assert_eq!(out.kind(), CompressorKind::Diff); + assert!(out.text().contains("reason=lockfile"), "{}", out.text()); + assert_eq!(store.get(out.token()).as_deref(), Some(s.as_str())); + } - // With the query: the auth run is preserved verbatim. - let with_q = compress(&s, false, Some("why is AUTHENTICATION_TOKEN failing")) - .expect("compresses") - .text; - assert!(with_q.contains("AUTHENTICATION_TOKEN"), "{with_q}"); - assert!( - with_q.contains("auth context 15"), - "pinned run must survive:\n{with_q}" - ); - // The unrelated other-file run still collapses, so output shrank. - assert!(with_q.contains("context line(s) omitted"), "{with_q}"); + #[test] + fn diff_noise_transform_skips_non_diff_input() { + let input = PipelineInput { + content: "plain text", + original_content: "plain text", + content_kind: ContentKind::PlainText, + original_bytes: "plain text".len(), + }; + let transform = DiffNoiseTransform::default(); + let store = MemoryCcrStore::default(); + + assert_eq!(transform.estimate_bloat(&input), 0.0); + assert!(transform.apply(&input, &store).is_none()); + } - // Without the query: the same auth run collapses. - let no_q = compress(&s, false, None).expect("compresses").text; - assert!( - !no_q.contains("auth context 15"), - "run should collapse:\n{no_q}" - ); + #[test] + fn non_diff_returns_none() { + assert!(compress("just some text\nno hunks here").is_none()); } } diff --git a/src/compressors/html.rs b/src/compressors/html.rs index e9f46fa..9a7a92a 100644 --- a/src/compressors/html.rs +++ b/src/compressors/html.rs @@ -10,7 +10,9 @@ use async_trait::async_trait; use super::Compressor; -use crate::types::{CompressInput, CompressOptions, CompressOutput, CompressorKind}; +use crate::cache::CcrStore; +use crate::pipeline::{OffloadOutput, OffloadTransform, PipelineInput, estimate_bloat}; +use crate::types::{CompressInput, CompressOptions, CompressOutput, CompressorKind, ContentKind}; /// Block-level tags after which we emit a newline so the extracted text keeps /// document structure (paragraphs, list items, headings, rows). @@ -54,6 +56,34 @@ const INLINE_TAGS: &[&str] = &[ pub struct HtmlCompressor; +/// Typed offload transform for lossy HTML-to-text extraction. +pub struct HtmlExtractTransform; + +impl OffloadTransform for HtmlExtractTransform { + fn name(&self) -> &'static str { + "html_extract" + } + + fn estimate_bloat(&self, input: &PipelineInput<'_>) -> f32 { + if input.content_kind != ContentKind::Html { + return 0.0; + } + f32::from(estimate_bloat(input.content, input.content_kind).score) / 100.0 + } + + fn apply(&self, input: &PipelineInput<'_>, store: &dyn CcrStore) -> Option { + if input.content_kind != ContentKind::Html { + return None; + } + let compacted = compress(input.content)?; + OffloadOutput::from_retained_put( + compacted.text, + CompressorKind::Html, + store.put(input.original_content), + ) + } +} + #[async_trait] impl Compressor for HtmlCompressor { fn kind(&self) -> CompressorKind { @@ -308,6 +338,8 @@ fn collapse_blank_lines(text: &str) -> String { #[cfg(test)] mod tests { use super::*; + use crate::cache::{CcrStore, MemoryCcrStore}; + use crate::pipeline::PipelineInput; #[test] fn strips_tags_and_scripts() { @@ -472,4 +504,47 @@ mod tests { assert!(out.text.contains(&format!("cell {i}")), "cell {i} kept"); } } + + #[test] + fn html_extract_transform_requires_retained_ccr() { + let mut html = String::from(""); + for i in 0..50 { + html.push_str(&format!( + "
cell {i}
" + )); + } + html.push_str(""); + let input = PipelineInput { + content: &html, + original_content: &html, + content_kind: ContentKind::Html, + original_bytes: html.len(), + }; + let transform = HtmlExtractTransform; + + let rejecting_store = MemoryCcrStore::new(1, 1); + assert!(transform.apply(&input, &rejecting_store).is_none()); + + let store = MemoryCcrStore::default(); + let out = transform.apply(&input, &store).expect("retained HTML"); + + assert_eq!(out.kind(), CompressorKind::Html); + assert!(out.text().contains("cell 7"), "{}", out.text()); + assert_eq!(store.get(out.token()).as_deref(), Some(html.as_str())); + } + + #[test] + fn html_extract_transform_skips_non_html_input() { + let input = PipelineInput { + content: "plain text", + original_content: "plain text", + content_kind: ContentKind::PlainText, + original_bytes: "plain text".len(), + }; + let transform = HtmlExtractTransform; + let store = MemoryCcrStore::default(); + + assert_eq!(transform.estimate_bloat(&input), 0.0); + assert!(transform.apply(&input, &store).is_none()); + } } diff --git a/src/compressors/json.rs b/src/compressors/json.rs index 627733f..7db9f84 100644 --- a/src/compressors/json.rs +++ b/src/compressors/json.rs @@ -1,479 +1,306 @@ -//! JSON crusher (SmartCrusher). +//! JSON-array crusher (SmartCrusher). //! //! Clean-room port of Headroom's `SmartCrusher` (Apache-2.0), extended with -//! variance-aware row preservation, nested-array discovery, column factoring and -//! query anchoring. An array of objects that repeat the same keys is the single -//! most common bloated tool output (API list responses, DB rows, search -//! manifests). Re-rendering it as a table emits each key **once** instead of per -//! row. +//! variance-aware row preservation. An array of objects that repeat the same +//! keys is the single most common bloated tool output (API list responses, DB +//! rows, search manifests). Re-rendering it as a table emits each key **once** +//! instead of per row. //! -//! On top of the base table: -//! * a top-level **object** is scanned breadth-first (depth ≤ 2) for the -//! largest uniform-object array, which is tabled under its JSON path with the -//! sibling scalars rendered as a preamble; -//! * **constant columns** (identical across every row) are hoisted into a -//! one-line note and dropped from the table (lossless); -//! * columns whose values are uniformly objects are **flattened** — the -//! name/description-like sub-keys are promoted to their own columns instead -//! of dumping an escaped-JSON blob per cell; -//! * plain **number/string arrays** collapse to distributional stats / -//! deduped counts; -//! * rows carrying **error text (nested), categorical rarities (Pareto), -//! rare structural fields, numeric outliers, or a query anchor** are always -//! kept when the table is row-dropped. -//! -//! The router offloads the full original to CCR, so everything dropped stays -//! recoverable. +//! Up to [`ROW_DROP_THRESHOLD`] rows every value is preserved (faithful +//! reformat). Above the threshold the table is row-dropped — but rather than a +//! blind head/tail window, rows carrying **error indicators** or **numeric +//! outliers** are always kept, so anomalies survive even when the bulk of a +//! homogeneous array is dropped. The router offloads the full original to CCR, +//! so the dropped rows stay recoverable. use async_trait::async_trait; use serde_json::{Map, Value}; -use std::collections::{BTreeSet, HashMap}; +use std::collections::{BTreeMap, BTreeSet, HashMap}; use std::fmt::Write as _; -use super::adaptive::compute_optimal_k; -use super::anchors::extract_anchors; +use super::Compressor; use super::signals::has_error_indicators; -use super::{BLOCK_NOTE, Compressor, block_token}; -use crate::types::{CompressInput, CompressOptions, CompressOutput, CompressorKind}; +use crate::cache::CcrStore; +use crate::pipeline::{ + OffloadOutput, OffloadTransform, PipelineInput, ReformatTransform, TransformOutput, + estimate_bloat, +}; +use crate::relevance::{Bm25Corpus, tokenize}; +use crate::types::{CompressInput, CompressOptions, CompressOutput, CompressorKind, ContentKind}; /// Minimum rows before tabular rendering is worth the header overhead. pub const MIN_ROWS: usize = 3; /// Above this many rows the table is additionally row-dropped. pub const ROW_DROP_THRESHOLD: usize = 40; -/// At most this many rows are kept from the head when row-dropping (the -/// adaptive selector may keep fewer for redundant tables). +/// Rows kept from the head when row-dropping. pub const HEAD_ROWS: usize = 20; -/// At most this many rows are kept from the tail when row-dropping (the -/// adaptive selector may keep fewer for redundant tables). +/// Rows kept from the tail when row-dropping. pub const TAIL_ROWS: usize = 10; -/// Adaptive floor: never keep fewer head+tail rows than this when the table -/// has more rows than it. -pub const MIN_KEPT_ROWS: usize = 8; /// Z-score beyond which a numeric cell is treated as an outlier worth keeping. pub const OUTLIER_SIGMA: f64 = 2.0; -/// At most this many outlier rows are kept per column (the most extreme -/// first), so a heavy-tailed column can't defeat the point of row-dropping. -pub const MAX_OUTLIERS_PER_COLUMN: usize = 8; -/// Cells longer than this are truncated (marking the table lossy). -pub const CELL_MAX: usize = 240; -/// Minimum elements before a plain number/string array is worth crushing. -pub const SCALAR_ARRAY_MIN: usize = 8; -/// A categorical column with more distinct values than this isn't Pareto-scanned. -pub const MAX_CARDINALITY: usize = 50; -/// Pareto coverage target: the smallest value set covering this fraction of rows. -pub const PARETO_COVER: f64 = 0.80; -/// Only rare-status rows are kept when the covering set is at most this size. -pub const PARETO_MAX_K: usize = 5; -/// Rows carrying a field present in fewer than this fraction of rows are kept. -pub const RARE_FIELD_RATIO: f64 = 0.20; +/// Delta multiplier beyond which a numeric transition is treated as a change point. +pub const CHANGE_POINT_DELTA_MULTIPLIER: f64 = 4.0; +/// Maximum numeric change-point rows to anchor per field. +pub const MAX_CHANGE_POINT_ANCHORS: usize = 20; +/// Maximum distinct values for a full-present field to be treated as a +/// discriminator bucket anchor. +pub const MAX_DISCRIMINATOR_BUCKETS: usize = 12; +/// Maximum exact duplicate row clusters to anchor while row-dropping. +pub const MAX_DUPLICATE_CLUSTER_ANCHORS: usize = 20; +/// Maximum near-duplicate row clusters to anchor while row-dropping. +pub const MAX_NEAR_DUPLICATE_CLUSTER_ANCHORS: usize = 20; +/// Maximum SimHash bit distance for rows to share a near-duplicate cluster. +pub const NEAR_DUPLICATE_SIMHASH_DISTANCE: u32 = 3; +/// Maximum evenly-spaced middle anchors to add for large generic arrays. +pub const MAX_SPREAD_ANCHORS: usize = 8; +/// Maximum extra spread anchors when semantic bigrams do not saturate early. +pub const MAX_ADAPTIVE_SPREAD_ANCHOR_BOOST: usize = 4; +/// Minimum distinct semantic bigrams before saturation changes anchor counts. +pub const MIN_SATURATION_BIGRAMS: usize = 16; +/// Share of distinct semantic bigrams that defines the saturation knee. +pub const SATURATION_KNEE_PERCENT: usize = 70; +/// Position threshold for treating the saturation knee as early. +pub const EARLY_SATURATION_POSITION_PERCENT: usize = 50; +/// Maximum information-dense middle rows to anchor. +pub const MAX_INFORMATION_DENSE_ANCHORS: usize = 8; +/// Extra positional rows to keep when query wording asks for a side of the list. +pub const QUERY_DIRECTION_EXTRA_ROWS: usize = 10; +/// Do not spend unbounded parse effort on huge string cells. +const MAX_STRINGIFIED_JSON_BYTES: usize = 16 * 1024; pub struct JsonCompressor; -#[async_trait] -impl Compressor for JsonCompressor { - fn kind(&self) -> CompressorKind { - CompressorKind::SmartCrusher - } +/// Typed reformat transform for faithful SmartCrusher table rendering. +#[derive(Debug, Clone, Default)] +pub struct SmartCrusherTableTransform; - async fn compress( - &self, - input: &CompressInput<'_>, - opts: &CompressOptions, - ) -> Option { - compress(input.content, opts.ccr_enabled, input.hint.query.as_deref()) - } +/// Typed offload transform for SmartCrusher row-dropping. +#[derive(Debug, Clone, Default)] +pub struct SmartCrusherRowsTransform { + query: Option, } -/// Compress a JSON payload. `block_tokens` offloads each omitted block to CCR so -/// its marker retrieves exactly that block. `query` (when present) yields exact- -/// match anchors that force any row mentioning them to survive row-dropping. -/// Returns `None` when there's nothing tabular/statistical to do or it wouldn't -/// shrink. -pub fn compress(content: &str, block_tokens: bool, query: Option<&str>) -> Option { - let value: Value = serde_json::from_str(content.trim()).ok()?; - let anchors = extract_anchors(query.unwrap_or_default()); +impl SmartCrusherRowsTransform { + pub fn new() -> Self { + Self::default() + } + + pub fn with_query(mut self, query: impl Into) -> Self { + self.query = Some(query.into()); + self + } - match &value { - Value::Array(array) => compress_top_array(&value, array, content, block_tokens, &anchors), - Value::Object(map) => compress_object(map, content, block_tokens, &anchors), - _ => None, + pub fn query(&self) -> Option<&str> { + self.query.as_deref() } } -/// Dispatch a top-level array to the object-table, number-stats or string-dedupe -/// crusher depending on its element kind. -fn compress_top_array( - value: &Value, - array: &[Value], - content: &str, - block_tokens: bool, - anchors: &[String], -) -> Option { - if array.len() >= MIN_ROWS && array.iter().all(Value::is_object) { - return object_table_output(value, array, content, block_tokens, anchors, None, ""); +impl ReformatTransform for SmartCrusherTableTransform { + fn name(&self) -> &'static str { + "smartcrusher_table" } - if array.len() >= SCALAR_ARRAY_MIN && array.iter().all(is_plain_number) { - return crush_number_array(array, content, block_tokens, ""); + + fn applies_to(&self, input: &PipelineInput<'_>) -> bool { + input.content_kind == ContentKind::Json } - if array.len() >= SCALAR_ARRAY_MIN && array.iter().all(Value::is_string) { - return crush_string_array(array, content, block_tokens, ""); + + fn apply(&self, input: &PipelineInput<'_>) -> Option { + if input.content_kind != ContentKind::Json { + return None; + } + let output = compress(input.content)?; + if output.lossy { + return None; + } + Some(TransformOutput::new( + output.text, + CompressorKind::SmartCrusher, + )) } - None } -/// A top-level object: find the largest uniform-object array nested within -/// (depth ≤ 2) and table it under its JSON path, else fall back to the largest -/// plain scalar array. Sibling scalars of the container become a preamble. -fn compress_object( - root: &Map, - content: &str, - block_tokens: bool, - anchors: &[String], -) -> Option { - let found = find_best_nested(root)?; - let container = found.container; - let key = found.path.rsplit('.').next().unwrap_or(&found.path); - let preamble = sibling_preamble(container, key, block_tokens); - - match found.kind { - CandidateKind::Objects => { - let arr = container.get(key)?.as_array()?; - object_table_output( - &Value::Null, - arr, - content, - block_tokens, - anchors, - Some(&found.path), - &preamble, - ) - } - CandidateKind::Numbers => { - let arr = container.get(key)?.as_array()?; - crush_number_array(arr, content, block_tokens, &found.path) - } - CandidateKind::Strings => { - let arr = container.get(key)?.as_array()?; - crush_string_array(arr, content, block_tokens, &found.path) +impl OffloadTransform for SmartCrusherRowsTransform { + fn name(&self) -> &'static str { + "smartcrusher_rows" + } + + fn estimate_bloat(&self, input: &PipelineInput<'_>) -> f32 { + if input.content_kind != ContentKind::Json { + return 0.0; + } + let score = f32::from(estimate_bloat(input.content, input.content_kind).score) / 100.0; + if self + .query + .as_deref() + .is_some_and(|query| !query.trim().is_empty()) + { + score.max(0.1) + } else { + score } } + + fn apply(&self, input: &PipelineInput<'_>, store: &dyn CcrStore) -> Option { + if input.content_kind != ContentKind::Json { + return None; + } + let output = compress_with_query(input.content, self.query())?; + if !output.lossy { + return None; + } + OffloadOutput::from_retained_put( + output.text, + CompressorKind::SmartCrusher, + store.put(input.original_content), + ) + } } -// --------------------------------------------------------------------------- -// Nested-array discovery -// --------------------------------------------------------------------------- +#[async_trait] +impl Compressor for JsonCompressor { + fn kind(&self) -> CompressorKind { + CompressorKind::SmartCrusher + } -#[derive(Clone, Copy, PartialEq)] -enum CandidateKind { - Objects, - Numbers, - Strings, + async fn compress( + &self, + input: &CompressInput<'_>, + _opts: &CompressOptions, + ) -> Option { + compress_with_query(input.content, input.hint.query.as_deref()) + } } -struct Candidate<'a> { - path: String, - container: &'a Map, - kind: CandidateKind, - rows: usize, +#[derive(Debug, Clone, PartialEq)] +pub struct JsonAnalysis { + pub row_count: usize, + pub columns: Vec, + pub fields: Vec, + pub estimated_table_bytes: usize, } -/// Breadth-first (depth ≤ 2) scan of `root` for the best crushable array. Prefers -/// the largest uniform-object array; failing that, the largest plain scalar -/// array. Returns `None` when nothing qualifies. -fn find_best_nested(root: &Map) -> Option> { - let mut candidates: Vec = Vec::new(); - collect_candidates(root, String::new(), 1, &mut candidates); - - // Object arrays win outright (most rows); scalar arrays are the fallback. - candidates - .iter() - .filter(|c| c.kind == CandidateKind::Objects) - .max_by_key(|c| c.rows) - .or_else(|| { - candidates - .iter() - .filter(|c| c.kind != CandidateKind::Objects) - .max_by_key(|c| c.rows) - }) - .map(|c| Candidate { - path: c.path.clone(), - container: c.container, - kind: c.kind, - rows: c.rows, - }) -} +impl JsonAnalysis { + /// Estimated bytes removed by rendering this payload as a SmartCrusher + /// table. This is a rough planning signal, not a benchmarked savings claim. + pub fn estimated_reduction_bytes(&self, original_bytes: usize) -> usize { + original_bytes.saturating_sub(self.estimated_table_bytes) + } -/// Visit the entries of `obj` at `depth`, recording any qualifying array and -/// recursing one level into nested objects (until `depth` exceeds 2). -fn collect_candidates<'a>( - obj: &'a Map, - prefix: String, - depth: usize, - out: &mut Vec>, -) { - for (key, val) in obj { - let path = if prefix.is_empty() { - key.clone() - } else { - format!("{prefix}.{key}") - }; - match val { - Value::Array(arr) => { - if let Some(kind) = classify_array(arr) { - out.push(Candidate { - path, - container: obj, - kind, - rows: arr.len(), - }); - } - } - Value::Object(inner) if depth < 2 => { - collect_candidates(inner, path, depth + 1, out); - } - _ => {} + /// Estimated fractional reduction from 0.0 to 1.0. Returns 0.0 for empty + /// originals or payloads that are not estimated to shrink. + pub fn estimated_reduction_ratio(&self, original_bytes: usize) -> f64 { + if original_bytes == 0 { + return 0.0; } + self.estimated_reduction_bytes(original_bytes) as f64 / original_bytes as f64 } } -/// Classify an array as a crushable object/number/string array, or `None`. -fn classify_array(arr: &[Value]) -> Option { - if arr.len() >= MIN_ROWS && arr.iter().all(Value::is_object) && union_columns(arr).len() >= 2 { - Some(CandidateKind::Objects) - } else if arr.len() >= SCALAR_ARRAY_MIN && arr.iter().all(is_plain_number) { - Some(CandidateKind::Numbers) - } else if arr.len() >= SCALAR_ARRAY_MIN && arr.iter().all(Value::is_string) { - Some(CandidateKind::Strings) - } else { - None - } +#[derive(Debug, Clone, PartialEq)] +pub struct JsonFieldAnalysis { + pub key: String, + pub present: usize, + pub types: BTreeSet<&'static str>, + pub unique_count: usize, + pub constant: bool, + pub sparse: bool, + pub numeric: Option, } -/// Render the scalar siblings of the tabled array key as `key: value` preamble -/// lines (truncated). Non-scalar siblings are skipped — the table/preamble is a -/// summary, and the router keeps the original recoverable. -fn sibling_preamble(container: &Map, array_key: &str, allow_truncate: bool) -> String { - let mut out = String::new(); - for (key, val) in container { - if key == array_key { - continue; +impl JsonFieldAnalysis { + /// Ratio of distinct rendered values among present cells for this field. + pub fn unique_ratio(&self) -> f64 { + if self.present == 0 { + return 0.0; } - let rendered = match val { - // Without CCR the preamble must stay faithful, so long sibling - // strings are kept whole rather than truncated. - Value::String(s) if allow_truncate => truncate_plain(s, CELL_MAX), - Value::String(s) => s.clone(), - Value::Bool(_) | Value::Number(_) | Value::Null => val.to_string(), - _ => continue, - }; - let _ = writeln!(out, "{key}: {rendered}"); + self.unique_count as f64 / self.present as f64 } - out } -// --------------------------------------------------------------------------- -// Object-array table -// --------------------------------------------------------------------------- - -/// Build the table and wrap it in a [`CompressOutput`], applying the minified- -/// JSON fallback only for a genuine top-level array (`minify_value` non-null). -#[allow(clippy::too_many_arguments)] -fn object_table_output( - minify_value: &Value, - array: &[Value], - content: &str, - block_tokens: bool, - anchors: &[String], - path: Option<&str>, - preamble: &str, -) -> Option { - let (table, lossy) = build_object_table(array, block_tokens, anchors, path, preamble)?; - - // Top-level arrays may still be better served by a straight minify; nested - // discovery always prefers the table (minifying the whole object defeats it). - if path.is_none() - && let Ok(minified) = serde_json::to_string(minify_value) - && minified.len() < content.len() - && minified.len() < table.len() - { - return Some(CompressOutput::reformatted( - minified, - CompressorKind::SmartCrusher, - )); - } - - if table.len() >= content.len() { - return None; - } - log::debug!( - "[tinyjuice][json] table {} rows, lossy={} ({} -> {} bytes)", - array.len(), - lossy, - content.len(), - table.len(), - ); - Some(if lossy { - CompressOutput::lossy(table, CompressorKind::SmartCrusher) - } else { - CompressOutput::reformatted(table, CompressorKind::SmartCrusher) - }) +#[derive(Debug, Clone, Copy, PartialEq)] +pub struct JsonNumericStats { + pub min: f64, + pub max: f64, + pub mean: f64, + pub stddev: f64, } -/// The output column plan for one source column. -enum ColPlan { - /// Identical across every row — hoisted to a preamble note, dropped. - Constant(String), - /// Rendered verbatim. - Plain, - /// Uniform-object column: promote `promoted` sub-keys to their own columns, - /// dump the rest as one compact-JSON blob column (when `remainder` non-empty). - Flat { - promoted: Vec, - remainder: Vec, - }, +#[derive(Debug, Clone)] +struct JsonPlan { + lossy: bool, + keep_rows: Vec, } -/// An output column resolved from the plans (owns its labels). -enum OutCol { - Plain(String), - FlatSub(String, String), - FlatBlob(String, Vec), +/// Compress a JSON array-of-objects into a compact table. Returns `None` when +/// the content isn't a uniform-enough array of objects or wouldn't shrink. +pub fn compress(content: &str) -> Option { + compress_with_query(content, None) } -/// Render an object-array as a markdown table (constant columns hoisted, uniform- -/// object columns flattened, cells truncated). Returns `(text, lossy)` or `None` -/// when there aren't ≥ 2 columns / any non-constant column left. -fn build_object_table( - array: &[Value], - block_tokens: bool, - anchors: &[String], - path: Option<&str>, - preamble: &str, -) -> Option<(String, bool)> { - let columns = union_columns(array); - if columns.len() < 2 { +/// Compress a JSON array-of-objects with optional query context for row anchors. +pub fn compress_with_query(content: &str, query: Option<&str>) -> Option { + let value: Value = serde_json::from_str(content.trim()).ok()?; + let array = value.as_array()?; + if array.len() < MIN_ROWS { return None; } - let n = array.len(); - - // Plan every source column, collecting constant-column notes and the ordered - // list of concrete output columns. - let mut notes: Vec<(String, String)> = Vec::new(); - let mut out_cols: Vec = Vec::new(); - for col in &columns { - match analyze_column(array, col, n) { - ColPlan::Constant(v) => notes.push((col.clone(), v)), - ColPlan::Plain => out_cols.push(OutCol::Plain(col.clone())), - ColPlan::Flat { - promoted, - remainder, - } => { - for sub in promoted { - out_cols.push(OutCol::FlatSub(col.clone(), sub)); - } - if !remainder.is_empty() { - out_cols.push(OutCol::FlatBlob(col.clone(), remainder)); - } - } - } + if !array.iter().all(Value::is_object) { + return None; } - if out_cols.is_empty() { + + let flat_rows = flatten_rows(array)?; + let analysis = analyze_flat_rows(&flat_rows); + let table_shape = table_shape(&flat_rows, &analysis); + let columns = table_shape.columns.clone(); + if columns.len() < 2 { return None; } - // Header labels. - let headers: Vec = out_cols - .iter() - .map(|oc| match oc { - OutCol::Plain(c) => escape_markdown_cell(c), - OutCol::FlatSub(c, s) => escape_markdown_cell(&format!("{c}.{s}")), - OutCol::FlatBlob(c, _) => escape_markdown_cell(&format!("{c}.\u{2026}")), - }) - .collect(); - let width = headers.len(); - - // Render each row's cells (tracking whether any cell was truncated). - let mut truncated = false; - let mut rows: Vec = Vec::with_capacity(n); - for item in array { - let obj = item.as_object()?; - let cells: Vec = out_cols + // Render every row's cells up front so we can choose full vs. row-dropped. + let mut rows: Vec = Vec::with_capacity(flat_rows.len()); + for obj in &flat_rows { + let cells: Vec = columns .iter() - .map(|oc| render_out_cell(obj, oc, block_tokens, &mut truncated)) + .map(|col| match obj.values.get(col) { + None => String::new(), + Some(v) => render_cell(v), + }) .collect(); - rows.push(render_markdown_row(&cells)); + rows.push(cells.join(" | ")); } - // Row-dropping (sampling the middle away) is information loss, so it only - // happens when CCR can recover the dropped rows. Without CCR the table - // keeps every row: a full markdown table is still far smaller than the - // pretty-printed JSON, and it is a faithful, lossless reshape. - let row_dropped = block_tokens && n > ROW_DROP_THRESHOLD; - let lossy = row_dropped || truncated; + let plan = plan_rows(&flat_rows, &analysis, query); + let mut out = String::with_capacity(content.len()); + let _ = writeln!( + out, + "[json table: {} rows × {} cols · blank=absent key · exact original via retrieve footer]", + rows.len(), + columns.len() + ); + write_constants_line(&mut out, &table_shape.constants); + let _ = writeln!(out, "{}", columns.join(" | ")); - // Assemble output: sibling preamble, constant notes, header, body. - let mut out = - String::with_capacity(preamble.len() + rows.iter().map(String::len).sum::()); - if !preamble.is_empty() { - out.push_str(preamble); - } - if !notes.is_empty() { - let joined = notes - .iter() - .map(|(k, v)| format!("{k}={v}")) - .collect::>() - .join(", "); - let _ = writeln!(out, "[all rows: {joined}]"); - } - // A lossy table samples rows/cells away and needs the retrieve footer for - // fidelity; a full table is a faithful reshape with nothing to recover. - let fidelity = if lossy { - "exact original via retrieve footer" - } else { - "faithful reshape — all rows and values preserved" - }; - match path { - Some(p) => { - let _ = writeln!( - out, - "[json table: {p} — {n} rows × {width} cols · blank=absent key · {fidelity}]" - ); - } - None => { - let _ = writeln!( - out, - "[json table: {n} rows × {width} cols · blank=absent key · {fidelity}]" - ); - } - } - let _ = writeln!(out, "{}", render_markdown_row(&headers)); - let _ = writeln!(out, "{}", render_markdown_separator(width)); - - let mut any_token = false; - if row_dropped { - let keep = rows_to_keep(array, &columns, n, anchors); - let mut gap_marker = |out: &mut String, start: usize, end: usize| { - let slice = serde_json::to_string(&array[start..end]).unwrap_or_default(); - let token = block_token(&slice, block_tokens && !slice.is_empty()); - any_token |= !token.is_empty(); - let _ = writeln!(out, "[... {} row(s) omitted ...{token}]", end - start); - }; + if plan.lossy { + // Keep head + tail PLUS any anomalous rows (errors / numeric outliers) + // so the signal in a large homogeneous array survives row-dropping. let mut prev: Option = None; - for &i in &keep { + for &i in &plan.keep_rows { if let Some(p) = prev { - if i > p + 1 { - gap_marker(&mut out, p + 1, i); + let gap = i - p - 1; + if gap > 0 { + let _ = writeln!(out, "[... {gap} row(s) omitted ...]"); } } else if i > 0 { - gap_marker(&mut out, 0, i); + let _ = writeln!(out, "[... {i} row(s) omitted ...]"); } let _ = writeln!(out, "{}", rows[i]); prev = Some(i); } - if let Some(p) = prev - && p + 1 < n - { - gap_marker(&mut out, p + 1, n); + if let Some(p) = prev { + let tail = rows.len().saturating_sub(p + 1); + if tail > 0 { + let _ = writeln!(out, "[... {tail} row(s) omitted ...]"); + } } } else { for row in &rows { @@ -481,564 +308,1040 @@ fn build_object_table( } } - let mut table = out.trim_end().to_string(); - if any_token { - table.push_str(BLOCK_NOTE); + let out = out.trim_end().to_string(); + if let Some(bucketed) = compress_shape_buckets(content, &flat_rows, query) + && bucketed.text.len() < out.len() + { + return Some(bucketed); + } + if out.len() >= content.len() { + return None; + } + log::debug!( + "[tokenjuice][json] {} rows × {} cols, lossy={} ({} -> {} bytes)", + rows.len(), + columns.len(), + plan.lossy, + content.len(), + out.len(), + ); + if plan.lossy { + Some(CompressOutput::lossy(out, CompressorKind::SmartCrusher)) + } else { + // All values preserved, but the array→table reformat changes layout. + Some(CompressOutput::reformatted( + out, + CompressorKind::SmartCrusher, + )) } - Some((table, lossy)) } -/// Plan a single source column: constant → hoisted, uniform-object with an -/// informative key → flattened, else plain. -fn analyze_column(array: &[Value], col: &str, n: usize) -> ColPlan { - let present: Vec<&Value> = array - .iter() - .filter_map(|item| item.as_object().and_then(|o| o.get(col))) - .collect(); +fn compress_shape_buckets( + original: &str, + rows: &[FlatRow], + query: Option<&str>, +) -> Option { + let buckets = shape_buckets(rows); + if buckets.len() < 2 { + return None; + } + + let mut out = String::with_capacity(original.len()); + let mut lossy = false; + let _ = write!( + out, + "[json bucketed tables: {} rows × {} buckets · __row=original row index · blank=absent key", + rows.len(), + buckets.len() + ); - // Constant: present in every row and all equal. - if present.len() == n && n > 0 { - let first = &present[0]; - if present.iter().all(|v| *v == *first) { - return ColPlan::Constant(compact_json(first)); + for bucket in &buckets { + if bucket.indices.len() < MIN_ROWS { + return None; + } + if bucket.columns.len() < 2 { + return None; } } - // Uniform object → maybe flatten. - if !present.is_empty() && present.iter().all(|v| v.is_object()) { - let subkeys = union_subkeys(&present); - let (promoted, remainder) = plan_flatten(&subkeys, &present); - if promoted.iter().any(|k| is_name_like(k) || is_desc_like(k)) { - return ColPlan::Flat { - promoted, - remainder, - }; + let bucket_rows = buckets + .iter() + .map(|bucket| { + bucket + .indices + .iter() + .map(|&index| rows[index].clone()) + .collect::>() + }) + .collect::>(); + let analyses = bucket_rows + .iter() + .map(|rows| analyze_flat_rows(rows)) + .collect::>(); + let table_shapes = bucket_rows + .iter() + .zip(&analyses) + .map(|(rows, analysis)| table_shape(rows, analysis)) + .collect::>(); + if table_shapes.iter().any(|shape| shape.columns.len() < 2) { + return None; + } + let plans = bucket_rows + .iter() + .zip(&analyses) + .map(|(rows, analysis)| plan_rows(rows, analysis, query)) + .collect::>(); + if plans.iter().any(|plan| plan.lossy) { + lossy = true; + out.push_str(" · exact original via retrieve footer"); + } + out.push_str("]\n"); + + for (bucket_index, bucket) in buckets.iter().enumerate() { + let table_shape = &table_shapes[bucket_index]; + let columns = &table_shape.columns; + let plan = &plans[bucket_index]; + let _ = writeln!( + out, + "\n[bucket {}: {} row(s) × {} cols]", + bucket_index + 1, + bucket.indices.len(), + columns.len() + ); + write_constants_line(&mut out, &table_shape.constants); + let _ = writeln!(out, "__row | {}", columns.join(" | ")); + + if plan.lossy { + let mut prev: Option = None; + for &local_index in &plan.keep_rows { + if let Some(previous) = prev { + let gap = local_index - previous - 1; + if gap > 0 { + let _ = writeln!(out, "[... {gap} row(s) omitted in bucket ...]"); + } + } else if local_index > 0 { + let _ = writeln!(out, "[... {local_index} row(s) omitted in bucket ...]"); + } + write_bucket_row(&mut out, rows, bucket.indices[local_index], columns); + prev = Some(local_index); + } + if let Some(previous) = prev { + let tail = bucket.indices.len().saturating_sub(previous + 1); + if tail > 0 { + let _ = writeln!(out, "[... {tail} row(s) omitted in bucket ...]"); + } + } + } else { + for &row_index in &bucket.indices { + write_bucket_row(&mut out, rows, row_index, columns); + } } } - ColPlan::Plain -} - -/// Choose the promoted sub-keys (name/id/title-like first, description-like -/// next, then up to three total filled with scalar-valued keys) and the -/// remainder to keep as a blob. -fn plan_flatten(subkeys: &[String], present: &[&Value]) -> (Vec, Vec) { - let mut promoted: Vec = Vec::new(); - if let Some(k) = subkeys.iter().find(|k| is_name_like(k)) { - promoted.push(k.clone()); + let out = out.trim_end().to_string(); + if out.len() >= original.len() { + return None; } - if let Some(k) = subkeys.iter().find(|k| is_desc_like(k)) { - promoted.push(k.clone()); + if lossy { + Some(CompressOutput::lossy(out, CompressorKind::SmartCrusher)) + } else { + Some(CompressOutput::reformatted( + out, + CompressorKind::SmartCrusher, + )) } - for k in subkeys { - if promoted.len() >= 3 { - break; - } - if !promoted.contains(k) && subkey_is_scalar(k, present) { - promoted.push(k.clone()); +} + +#[derive(Debug)] +struct ShapeBucket { + columns: Vec, + indices: Vec, +} + +fn shape_buckets(rows: &[FlatRow]) -> Vec { + let mut buckets: Vec = Vec::new(); + for (index, row) in rows.iter().enumerate() { + let columns = row.values.keys().cloned().collect::>(); + if let Some(bucket) = buckets.iter_mut().find(|bucket| bucket.columns == columns) { + bucket.indices.push(index); + } else { + buckets.push(ShapeBucket { + columns, + indices: vec![index], + }); } } - let remainder: Vec = subkeys + buckets +} + +fn write_bucket_row(out: &mut String, rows: &[FlatRow], row_index: usize, columns: &[String]) { + let row = &rows[row_index]; + let cells = columns .iter() - .filter(|k| !promoted.contains(k)) - .cloned() - .collect(); - (promoted, remainder) + .map(|col| row.values.get(col).map(render_cell).unwrap_or_default()) + .collect::>(); + let _ = writeln!(out, "{row_index} | {}", cells.join(" | ")); } -/// True if the sub-key's value is scalar in the first object that carries it. -fn subkey_is_scalar(sub: &str, present: &[&Value]) -> bool { - for v in present { - if let Some(obj) = v.as_object() - && let Some(inner) = obj.get(sub) - { - return !matches!(inner, Value::Object(_) | Value::Array(_)); +#[derive(Debug, Clone)] +struct TableShape { + columns: Vec, + constants: Vec<(String, String)>, +} + +fn table_shape(rows: &[FlatRow], analysis: &JsonAnalysis) -> TableShape { + let mut constants = Vec::new(); + let mut columns = Vec::new(); + for field in &analysis.fields { + if field.constant { + if let Some(value) = rows + .first() + .and_then(|row| row.values.get(&field.key)) + .map(render_cell) + { + constants.push((field.key.clone(), value)); + } else { + columns.push(field.key.clone()); + } + } else { + columns.push(field.key.clone()); } } - false + + if columns.len() < 2 { + TableShape { + columns: analysis.columns.clone(), + constants: Vec::new(), + } + } else { + TableShape { columns, constants } + } } -/// Render one output cell for a row. Truncates over-long cells only when -/// `allow_truncate` is set (CCR can recover the dropped tail); otherwise the -/// full value is kept so the table stays a faithful, information-preserving -/// reshape. -fn render_out_cell( - obj: &Map, - oc: &OutCol, - allow_truncate: bool, - truncated: &mut bool, -) -> String { - let raw = match oc { - OutCol::Plain(col) => obj.get(col).map(render_cell).unwrap_or_default(), - OutCol::FlatSub(col, sub) => obj - .get(col) - .and_then(Value::as_object) - .and_then(|o| o.get(sub)) - .map(render_cell) - .unwrap_or_default(), - OutCol::FlatBlob(col, subs) => match obj.get(col).and_then(Value::as_object) { - Some(o) => { - let mut kept = Map::new(); - for s in subs { - if let Some(v) = o.get(s) { - kept.insert(s.clone(), v.clone()); - } - } - if kept.is_empty() { - String::new() - } else { - escape_markdown_cell(&compact_json(&Value::Object(kept))) - } - } - None => obj.get(col).map(render_cell).unwrap_or_default(), - }, - }; - if !allow_truncate { - // No CCR to recover a cut cell — keep the full value (lossless). - return raw; +fn write_constants_line(out: &mut String, constants: &[(String, String)]) { + if constants.is_empty() { + return; } - let (cell, cut) = truncate_cell(&raw); - *truncated |= cut; - cell + let rendered = constants + .iter() + .map(|(key, value)| format!("{key}={value}")) + .collect::>() + .join(" | "); + let _ = writeln!(out, "constants: {rendered}"); } -// --------------------------------------------------------------------------- -// Row-keeping (row-drop must-keeps) -// --------------------------------------------------------------------------- +/// Analyze a JSON array-of-objects without rendering it. +pub fn analyze(content: &str) -> Option { + let value: Value = serde_json::from_str(content.trim()).ok()?; + let array = value.as_array()?; + if array.len() < MIN_ROWS || !array.iter().all(Value::is_object) { + return None; + } + let flat_rows = flatten_rows(array)?; + Some(analyze_flat_rows(&flat_rows)) +} + +/// Pick the row indices to keep when row-dropping: the head/tail windows plus +/// rows flagged as anomalous, numeric change points, duplicate-cluster +/// representatives, spread anchors, information-dense rows, query-direction +/// anchors, representative discriminator buckets, or query-relevant. Returns +/// ascending, de-duplicated indices. +fn plan_rows(rows: &[FlatRow], analysis: &JsonAnalysis, query: Option<&str>) -> JsonPlan { + let n = rows.len(); + let lossy = n > ROW_DROP_THRESHOLD; + if !lossy { + return JsonPlan { + lossy: false, + keep_rows: (0..n).collect(), + }; + } -/// Pick the row indices to keep when row-dropping: adaptively sized head/tail -/// windows plus every anomalous row — nested error text, numeric outliers, -/// Pareto-rare categorical values, rare structural fields, and query-anchor -/// hits. Ascending, de-duped. -fn rows_to_keep(array: &[Value], columns: &[String], n: usize, anchors: &[String]) -> Vec { - let (head, tail) = adaptive_head_tail(array); let mut keep: BTreeSet = BTreeSet::new(); - for i in 0..head.min(n) { + for i in 0..dynamic_head_rows(n) { keep.insert(i); } - for i in n.saturating_sub(tail)..n { + for i in n.saturating_sub(dynamic_tail_rows(n))..n { + keep.insert(i); + } + for i in spread_anchor_rows(rows, &keep) { + keep.insert(i); + } + for i in duplicate_cluster_representatives(rows) { + keep.insert(i); + } + for i in near_duplicate_cluster_representatives(rows, &keep) { + keep.insert(i); + } + for i in information_dense_rows(rows, &keep) { + keep.insert(i); + } + for i in query_direction_rows(n, query) { keep.insert(i); } - // (2) Error text, scanning nested object/array values to depth 2. - for (i, item) in array.iter().enumerate() { - if value_has_error(item, 2) { + // Error-text rows: any string/scalar cell carrying an error indicator. + for (i, row) in rows.iter().enumerate() { + let row_has_error = row.values.values().any(|v| match v { + Value::String(s) => has_error_indicators(s), + other => has_error_indicators(&other.to_string()), + }); + if row_has_error { keep.insert(i); } } - // (8) Query anchors: keep any row whose serialized form contains an anchor. - if !anchors.is_empty() { - for (i, item) in array.iter().enumerate() { - let serialized = compact_json(item).to_ascii_lowercase(); - if anchors.iter().any(|a| serialized.contains(a)) { + // Numeric outliers: for each column, compute mean/std over numeric cells and + // keep rows whose value is beyond OUTLIER_SIGMA. Bounded by a cap so a wide + // anomalous tail can't defeat the point of dropping. + for field in &analysis.fields { + if let Some(stats) = field.numeric + && stats.stddev > f64::EPSILON + { + for (i, row) in rows.iter().enumerate() { + if let Some(x) = row.values.get(&field.key).and_then(Value::as_f64) + && ((x - stats.mean) / stats.stddev).abs() >= OUTLIER_SIGMA + { + keep.insert(i); + } + } + } + if field.numeric.is_some() { + for i in numeric_change_point_rows(rows, &field.key) { keep.insert(i); } } + if field.sparse { + for (i, row) in rows.iter().enumerate() { + if row.values.contains_key(&field.key) { + keep.insert(i); + } + } + } + if is_discriminator_field(field, rows.len()) { + let mut seen_values = BTreeSet::new(); + for (i, row) in rows.iter().enumerate() { + if let Some(value) = row.values.get(&field.key) + && seen_values.insert(render_cell(value)) + { + keep.insert(i); + } + } + } } - for col in columns { - keep_numeric_outliers(array, col, &mut keep); - keep_pareto_rare(array, col, n, &mut keep); - keep_rare_field(array, col, n, &mut keep); + if let Some(query) = query.filter(|q| !q.trim().is_empty()) { + let docs: Vec = rows.iter().map(row_text).collect(); + let corpus = Bm25Corpus::new(docs.iter().map(String::as_str)); + let mut scored = corpus.score_all(query); + scored.sort_by(|a, b| { + b.score + .partial_cmp(&a.score) + .unwrap_or(std::cmp::Ordering::Equal) + .then_with(|| a.index.cmp(&b.index)) + }); + for score in scored + .into_iter() + .take(20) + .filter(|score| score.score > 0.0) + { + keep.insert(score.index); + } } - keep.into_iter().collect() + JsonPlan { + lossy, + keep_rows: keep.into_iter().collect(), + } } -/// Size the head/tail keep windows from the rows themselves: highly redundant -/// arrays collapse toward [`MIN_KEPT_ROWS`], diverse ones keep up to the full -/// [`HEAD_ROWS`] + [`TAIL_ROWS`] budget. The total splits ~2/3 head (first -/// rows) and ~1/3 tail (last rows). -fn adaptive_head_tail(array: &[Value]) -> (usize, usize) { - let serialized: Vec = array.iter().map(compact_json).collect(); - let refs: Vec<&str> = serialized.iter().map(String::as_str).collect(); - let k = compute_optimal_k(&refs, MIN_KEPT_ROWS, HEAD_ROWS + TAIL_ROWS); - let tail = (k / 3).max(1); - (k - tail, tail) +fn is_discriminator_field(field: &JsonFieldAnalysis, row_count: usize) -> bool { + field.present == row_count + && field.numeric.is_none() + && (2..=MAX_DISCRIMINATOR_BUCKETS).contains(&field.unique_count) } -/// Recursively test for error indicators in string leaves down to `depth`. -fn value_has_error(v: &Value, depth: usize) -> bool { - match v { - Value::String(s) => has_error_indicators(s), - Value::Array(a) if depth > 0 => a.iter().any(|x| value_has_error(x, depth - 1)), - Value::Object(m) if depth > 0 => m.values().any(|x| value_has_error(x, depth - 1)), - _ => false, +fn duplicate_cluster_representatives(rows: &[FlatRow]) -> Vec { + let mut clusters: BTreeMap = BTreeMap::new(); + for (i, row) in rows.iter().enumerate() { + let entry = clusters.entry(row_text(row)).or_insert((i, 0)); + entry.1 += 1; + } + + let mut repeated = clusters + .into_values() + .filter(|(_, count)| *count > 1) + .collect::>(); + repeated.sort_by(|(a_first, a_count), (b_first, b_count)| { + b_count.cmp(a_count).then_with(|| a_first.cmp(b_first)) + }); + repeated + .into_iter() + .take(MAX_DUPLICATE_CLUSTER_ANCHORS) + .map(|(first, _)| first) + .collect() +} + +#[derive(Debug, Clone)] +struct NearDuplicateCluster { + first: usize, + count: usize, + fingerprint: u64, +} + +fn near_duplicate_cluster_representatives( + rows: &[FlatRow], + existing_keep: &BTreeSet, +) -> Vec { + let mut clusters: Vec = Vec::new(); + for (i, row) in rows.iter().enumerate() { + let fingerprint = semantic_simhash(&row_text(row)); + if fingerprint == 0 { + continue; + } + if let Some(cluster) = clusters.iter_mut().find(|cluster| { + (cluster.fingerprint ^ fingerprint).count_ones() <= NEAR_DUPLICATE_SIMHASH_DISTANCE + }) { + cluster.count += 1; + } else { + clusters.push(NearDuplicateCluster { + first: i, + count: 1, + fingerprint, + }); + } } + + clusters.retain(|cluster| cluster.count > 1 && !existing_keep.contains(&cluster.first)); + clusters.sort_by(|a, b| b.count.cmp(&a.count).then_with(|| a.first.cmp(&b.first))); + clusters + .into_iter() + .take(MAX_NEAR_DUPLICATE_CLUSTER_ANCHORS) + .map(|cluster| cluster.first) + .collect() } -/// (existing) Keep the most extreme numeric outliers in `col`. -fn keep_numeric_outliers(array: &[Value], col: &str, keep: &mut BTreeSet) { - let nums: Vec<(usize, f64)> = array +fn spread_anchor_rows(rows: &[FlatRow], existing_keep: &BTreeSet) -> Vec { + let n = rows.len(); + let start = dynamic_head_rows(n).min(n); + let end = n.saturating_sub(dynamic_tail_rows(n)); + if start >= end { + return Vec::new(); + } + + let middle_len = end - start; + let anchor_count = adaptive_spread_anchor_count(rows, start, end); + if anchor_count == 0 { + return Vec::new(); + } + + let mut seen = existing_keep .iter() - .enumerate() - .filter_map(|(i, item)| { - item.as_object() - .and_then(|o| o.get(col)) - .and_then(Value::as_f64) - .map(|x| (i, x)) - }) - .collect(); - if nums.len() < 4 { - return; + .filter_map(|&i| rows.get(i).map(row_text)) + .collect::>(); + let mut anchors = Vec::new(); + for rank in 1..=anchor_count { + let target = start + (rank * (middle_len + 1)) / (anchor_count + 1); + if let Some(index) = nearest_unseen_row(rows, start, end, target, &mut seen) { + anchors.push(index); + } } - let mean = nums.iter().map(|(_, x)| x).sum::() / nums.len() as f64; - let var = nums.iter().map(|(_, x)| (x - mean).powi(2)).sum::() / nums.len() as f64; - let std = var.sqrt(); - if std <= f64::EPSILON { - return; + anchors.sort_unstable(); + anchors +} + +fn adaptive_spread_anchor_count(rows: &[FlatRow], start: usize, end: usize) -> usize { + let middle_len = end - start; + let base = (middle_len / ROW_DROP_THRESHOLD).min(MAX_SPREAD_ANCHORS); + if base == 0 { + return 0; } - let mut outliers: Vec<(usize, f64)> = nums - .into_iter() - .map(|(i, x)| (i, ((x - mean) / std).abs())) - .filter(|(_, z)| *z >= OUTLIER_SIGMA) - .collect(); - outliers.sort_by(|a, b| b.1.partial_cmp(&a.1).unwrap_or(std::cmp::Ordering::Equal)); - for (i, _) in outliers.into_iter().take(MAX_OUTLIERS_PER_COLUMN) { - keep.insert(i); + + let total = cumulative_semantic_bigrams(rows, start, end); + if total < MIN_SATURATION_BIGRAMS { + return base; + } + if semantic_bigrams_saturate_early(rows, start, end, total) { + return base; } + + let boost = base.clamp(1, MAX_ADAPTIVE_SPREAD_ANCHOR_BOOST); + (base + boost).min(MAX_SPREAD_ANCHORS) +} + +fn semantic_bigrams_saturate_early( + rows: &[FlatRow], + start: usize, + end: usize, + total: usize, +) -> bool { + semantic_bigram_knee_percent(rows, start, end, total) + .is_some_and(|position| position <= EARLY_SATURATION_POSITION_PERCENT) } -/// (5) Pareto rare-status: for a low-cardinality string column, keep rows whose -/// value falls outside the smallest ≥ 80%-covering set when that set is tiny. -fn keep_pareto_rare(array: &[Value], col: &str, n: usize, keep: &mut BTreeSet) { - let mut freq: HashMap<&str, usize> = HashMap::new(); - let mut total = 0usize; - for item in array { - if let Some(Value::String(s)) = item.as_object().and_then(|o| o.get(col)) { - *freq.entry(s.as_str()).or_default() += 1; - total += 1; +fn semantic_bigram_knee_percent( + rows: &[FlatRow], + start: usize, + end: usize, + total: usize, +) -> Option { + let threshold = total * SATURATION_KNEE_PERCENT; + let middle_len = end - start; + let mut seen = BTreeSet::new(); + for (offset, row) in rows[start..end].iter().enumerate() { + let tokens = semantic_tokens(&row_text(row)); + for pair in tokens.windows(2) { + seen.insert((pair[0].clone(), pair[1].clone())); + } + if seen.len() * 100 >= threshold { + let consumed = offset + 1; + return Some((consumed * 100).div_ceil(middle_len)); } } - // Require a genuinely categorical column that most rows participate in. - if total < n / 2 || freq.len() < 2 || freq.len() > MAX_CARDINALITY { - return; - } - let mut counts: Vec<(&str, usize)> = freq.into_iter().collect(); - counts.sort_by(|a, b| b.1.cmp(&a.1).then(a.0.cmp(b.0))); - let target = (PARETO_COVER * total as f64).ceil() as usize; - let mut covered = 0usize; - let mut k = 0usize; - let mut common: BTreeSet<&str> = BTreeSet::new(); - for (val, c) in &counts { - if covered >= target { - break; + None +} + +fn cumulative_semantic_bigrams(rows: &[FlatRow], start: usize, end: usize) -> usize { + let mut seen = BTreeSet::new(); + for row in &rows[start..end] { + let tokens = semantic_tokens(&row_text(row)); + for pair in tokens.windows(2) { + seen.insert((pair[0].clone(), pair[1].clone())); } - common.insert(val); - covered += c; - k += 1; } - // Only fire when a tiny head covers the bulk AND rare values remain. - if k > PARETO_MAX_K || common.len() >= counts.len() { - return; + seen.len() +} + +fn semantic_tokens(text: &str) -> Vec { + tokenize(text) + .into_iter() + .filter(|token| !token.chars().any(|ch| ch.is_ascii_digit())) + .collect() +} + +fn semantic_simhash(text: &str) -> u64 { + let tokens = semantic_tokens(text); + if tokens.is_empty() { + return 0; } - for (i, item) in array.iter().enumerate() { - if let Some(Value::String(s)) = item.as_object().and_then(|o| o.get(col)) - && !common.contains(s.as_str()) - { - keep.insert(i); + + let mut buckets = [0i32; 64]; + for token in tokens { + let hash = stable_token_hash(&token); + for (bit, bucket) in buckets.iter_mut().enumerate() { + if (hash >> bit) & 1 == 1 { + *bucket += 1; + } else { + *bucket -= 1; + } } } -} -/// (6) Rare structural field: if `col` appears in < 20% of rows, keep the rows -/// that carry it. -fn keep_rare_field(array: &[Value], col: &str, n: usize, keep: &mut BTreeSet) { - let present: Vec = array - .iter() + buckets + .into_iter() .enumerate() - .filter(|(_, item)| item.as_object().is_some_and(|o| o.contains_key(col))) - .map(|(i, _)| i) - .collect(); - let threshold = (RARE_FIELD_RATIO * n as f64).floor() as usize; - if present.is_empty() || present.len() >= threshold.max(1) { - return; - } - for i in present { - keep.insert(i); + .fold(0u64, |fingerprint, (bit, score)| { + if score > 0 { + fingerprint | (1u64 << bit) + } else { + fingerprint + } + }) +} + +fn stable_token_hash(token: &str) -> u64 { + let mut hash = 0xcbf29ce484222325u64; + for byte in token.as_bytes() { + hash ^= u64::from(*byte); + hash = hash.wrapping_mul(0x100000001b3); } + hash } -// --------------------------------------------------------------------------- -// Scalar-array crushers -// --------------------------------------------------------------------------- +fn nearest_unseen_row( + rows: &[FlatRow], + start: usize, + end: usize, + target: usize, + seen: &mut BTreeSet, +) -> Option { + for offset in 0.. { + let left = target.checked_sub(offset).filter(|&i| i >= start); + if let Some(i) = left + && seen.insert(row_text(&rows[i])) + { + return Some(i); + } -/// Collapse a plain-number array to distributional stats + head/tail samples. -fn crush_number_array( - arr: &[Value], - content: &str, - block_tokens: bool, - path: &str, -) -> Option { - let mut nums: Vec = arr.iter().filter_map(Value::as_f64).collect(); - if nums.len() < SCALAR_ARRAY_MIN { - return None; + let right = target + offset; + if right < end && Some(right) != left && seen.insert(row_text(&rows[right])) { + return Some(right); + } + + if left.is_none() && right >= end { + break; + } } - let n = nums.len(); - nums.sort_by(|a, b| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal)); - let mean = nums.iter().sum::() / n as f64; - let pct = |q: f64| nums[((q * (n - 1) as f64).round() as usize).min(n - 1)]; + None +} - let head = 5.min(n); - let tail = 3.min(n.saturating_sub(head)); - let mut out = String::new(); - let label = if path.is_empty() { - String::new() - } else { - format!(" {path}") - }; - let _ = writeln!( - out, - "[json number array{label}: {n} values · min={} p25={} median={} p75={} max={} mean={}]", - fmt_num(pct(0.0)), - fmt_num(pct(0.25)), - fmt_num(pct(0.5)), - fmt_num(pct(0.75)), - fmt_num(pct(1.0)), - fmt_num(mean), - ); - let head_vals: Vec = arr[..head].iter().map(compact_json).collect(); - let _ = writeln!(out, "head: {}", head_vals.join(", ")); - if head + tail < arr.len() { - let slice = serde_json::to_string(&arr[head..arr.len() - tail]).unwrap_or_default(); - let token = block_token(&slice, block_tokens && !slice.is_empty()); - let _ = writeln!( - out, - "[... {} value(s) omitted ...{token}]", - arr.len() - head - tail - ); - if !token.is_empty() { - out.push_str(BLOCK_NOTE); - out.push('\n'); +fn numeric_change_point_rows(rows: &[FlatRow], key: &str) -> Vec { + let mut deltas: Vec<(usize, f64)> = Vec::new(); + let mut prev: Option = None; + for (i, row) in rows.iter().enumerate() { + let Some(current) = row.values.get(key).and_then(Value::as_f64) else { + prev = None; + continue; + }; + if let Some(previous) = prev { + let delta = (current - previous).abs(); + if delta > f64::EPSILON { + deltas.push((i, delta)); + } } + prev = Some(current); } - if tail > 0 { - let tail_vals: Vec = arr[arr.len() - tail..].iter().map(compact_json).collect(); - let _ = writeln!(out, "tail: {}", tail_vals.join(", ")); + if deltas.is_empty() { + return Vec::new(); } - let text = out.trim_end().to_string(); - if text.len() >= content.len() { - return None; - } - Some(CompressOutput::lossy(text, CompressorKind::SmartCrusher)) + let baseline = if deltas.len() >= 3 { + median_delta(deltas.iter().map(|(_, delta)| *delta).collect()) + } else { + f64::EPSILON + }; + let threshold = if baseline > f64::EPSILON { + baseline * CHANGE_POINT_DELTA_MULTIPLIER + } else { + f64::EPSILON + }; + + deltas.sort_by(|(a_index, a_delta), (b_index, b_delta)| { + b_delta + .partial_cmp(a_delta) + .unwrap_or(std::cmp::Ordering::Equal) + .then_with(|| a_index.cmp(b_index)) + }); + let mut anchors = deltas + .into_iter() + .filter(|(_, delta)| *delta >= threshold) + .take(MAX_CHANGE_POINT_ANCHORS) + .map(|(index, _)| index) + .collect::>(); + anchors.sort_unstable(); + anchors } -/// Collapse a plain-string array to deduped value/count pairs + head/tail. -fn crush_string_array( - arr: &[Value], - content: &str, - block_tokens: bool, - path: &str, -) -> Option { - let strings: Vec<&str> = arr.iter().filter_map(Value::as_str).collect(); - if strings.len() < SCALAR_ARRAY_MIN { - return None; +fn median_delta(mut deltas: Vec) -> f64 { + deltas.sort_by(|a, b| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal)); + deltas[deltas.len() / 2] +} + +fn information_dense_rows(rows: &[FlatRow], existing_keep: &BTreeSet) -> Vec { + let n = rows.len(); + let start = dynamic_head_rows(n).min(n); + let end = n.saturating_sub(dynamic_tail_rows(n)); + if start >= end { + return Vec::new(); } - let n = strings.len(); - let mut freq: HashMap<&str, usize> = HashMap::new(); - let mut order: Vec<&str> = Vec::new(); - for s in &strings { - if !freq.contains_key(s) { - order.push(s); - } - *freq.entry(s).or_default() += 1; + + let seen = existing_keep + .iter() + .filter_map(|&i| rows.get(i).map(row_text)) + .collect::>(); + let mut scored = (start..end) + .filter(|index| !existing_keep.contains(index)) + .filter_map(|index| { + let text = row_text(&rows[index]); + if seen.contains(&text) { + return None; + } + let unique_terms = tokenize(&text).into_iter().collect::>().len(); + (unique_terms > 0).then_some((index, unique_terms, text.len())) + }) + .collect::>(); + if scored.is_empty() { + return Vec::new(); } - let unique = order.len(); - let mut counts: Vec<(&str, usize)> = order.iter().map(|s| (*s, freq[s])).collect(); - counts.sort_by(|a, b| b.1.cmp(&a.1).then(a.0.cmp(b.0))); - const MAX_LISTED: usize = 12; + let baseline = median_usize(scored.iter().map(|(_, terms, _)| *terms).collect()); + let threshold = baseline + (baseline / 2).max(2); - let label = if path.is_empty() { - String::new() - } else { - format!(" {path}") + scored.sort_by(|(a_index, a_terms, a_len), (b_index, b_terms, b_len)| { + b_terms + .cmp(a_terms) + .then_with(|| b_len.cmp(a_len)) + .then_with(|| a_index.cmp(b_index)) + }); + let mut anchors = scored + .into_iter() + .filter(|(_, terms, _)| *terms > threshold) + .take(MAX_INFORMATION_DENSE_ANCHORS) + .map(|(index, _, _)| index) + .collect::>(); + anchors.sort_unstable(); + anchors +} + +fn median_usize(mut values: Vec) -> usize { + values.sort_unstable(); + values[values.len() / 2] +} + +fn query_direction_rows(row_count: usize, query: Option<&str>) -> Vec { + let Some(direction) = query.and_then(query_direction) else { + return Vec::new(); }; - let mut out = String::new(); - let _ = writeln!( - out, - "[json string array{label}: {n} values · {unique} unique]" - ); - for (val, c) in counts.iter().take(MAX_LISTED) { - let (cell, _) = truncate_cell(val); - let _ = writeln!(out, "{} ×{c}", compact_json(&Value::String(cell))); - } - if unique > MAX_LISTED { - let slice = serde_json::to_string(arr).unwrap_or_default(); - let token = block_token(&slice, block_tokens && !slice.is_empty()); - let _ = writeln!( - out, - "[... {} more unique value(s) ...{token}]", - unique - MAX_LISTED - ); - if !token.is_empty() { - out.push_str(BLOCK_NOTE); - out.push('\n'); + match direction { + QueryDirection::Front => { + let start = dynamic_head_rows(row_count).min(row_count); + let end = (start + QUERY_DIRECTION_EXTRA_ROWS).min(row_count); + (start..end).collect() + } + QueryDirection::Back => { + let tail_start = row_count.saturating_sub(dynamic_tail_rows(row_count)); + let start = tail_start.saturating_sub(QUERY_DIRECTION_EXTRA_ROWS); + (start..tail_start).collect() } } +} - let text = out.trim_end().to_string(); - if text.len() >= content.len() { - return None; +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum QueryDirection { + Front, + Back, +} + +fn query_direction(query: &str) -> Option { + let tokens = tokenize(query).into_iter().collect::>(); + if tokens.iter().any(|token| { + matches!( + token.as_str(), + "latest" | "recent" | "current" | "newest" | "last" + ) + }) { + return Some(QueryDirection::Back); + } + if tokens.iter().any(|token| { + matches!( + token.as_str(), + "first" | "oldest" | "earliest" | "initial" | "beginning" + ) + }) { + return Some(QueryDirection::Front); } - Some(CompressOutput::lossy(text, CompressorKind::SmartCrusher)) + None } -// --------------------------------------------------------------------------- -// Helpers -// --------------------------------------------------------------------------- - -/// First-seen union of object keys across an array of objects (stable order). -fn union_columns(array: &[Value]) -> Vec { - let mut columns: Vec = Vec::new(); - for item in array { - if let Some(obj) = item.as_object() { - for key in obj.keys() { - if !columns.iter().any(|c| c == key) { - columns.push(key.clone()); - } - } - } +#[derive(Debug, Clone)] +struct FlatRow { + values: BTreeMap, +} + +fn flatten_rows(array: &[Value]) -> Option> { + array + .iter() + .map(|item| { + let obj = item.as_object()?; + Some(FlatRow { + values: flatten_object(obj), + }) + }) + .collect() +} + +fn flatten_object(obj: &Map) -> BTreeMap { + let mut out = BTreeMap::new(); + for (key, value) in obj { + flatten_value(key, value, &mut out); } - columns + out } -/// First-seen union of sub-keys across a set of object values (stable order). -fn union_subkeys(values: &[&Value]) -> Vec { - let mut keys: Vec = Vec::new(); - for v in values { - if let Some(obj) = v.as_object() { - for key in obj.keys() { - if !keys.iter().any(|c| c == key) { - keys.push(key.clone()); +fn flatten_value(prefix: &str, value: &Value, out: &mut BTreeMap) { + match value { + Value::Object(obj) if !obj.is_empty() => { + for (key, child) in obj { + flatten_value(&format!("{prefix}.{key}"), child, out); + } + } + Value::String(text) => { + if let Some(parsed) = parse_stringified_json(text) { + match parsed { + Value::Object(obj) if !obj.is_empty() => { + for (key, child) in &obj { + flatten_value(&format!("{prefix}.{key}"), child, out); + } + } + Value::Array(_) => { + out.insert(prefix.to_string(), parsed); + } + _ => { + out.insert(prefix.to_string(), value.clone()); + } } + } else { + out.insert(prefix.to_string(), value.clone()); } } + _ => { + out.insert(prefix.to_string(), value.clone()); + } } - keys } -fn is_name_like(key: &str) -> bool { - let k = key.to_ascii_lowercase(); - matches!( - k.as_str(), - "name" | "id" | "title" | "key" | "slug" | "label" | "identifier" - ) +fn parse_stringified_json(text: &str) -> Option { + let trimmed = text.trim(); + if trimmed.len() > MAX_STRINGIFIED_JSON_BYTES { + return None; + } + if !(trimmed.starts_with('{') || trimmed.starts_with('[')) { + return None; + } + serde_json::from_str(trimmed).ok() } -fn is_desc_like(key: &str) -> bool { - let k = key.to_ascii_lowercase(); - matches!( - k.as_str(), - "description" | "desc" | "summary" | "comment" | "text" | "message" | "detail" - ) -} +fn analyze_flat_rows(rows: &[FlatRow]) -> JsonAnalysis { + let mut present: HashMap = HashMap::new(); + let mut types: HashMap> = HashMap::new(); + let mut uniques: HashMap> = HashMap::new(); + let mut numeric_values: HashMap> = HashMap::new(); + + for row in rows { + for (key, value) in &row.values { + *present.entry(key.clone()).or_insert(0) += 1; + types + .entry(key.clone()) + .or_default() + .insert(value_type(value)); + uniques + .entry(key.clone()) + .or_default() + .insert(render_cell(value)); + if let Some(x) = value.as_f64() { + numeric_values.entry(key.clone()).or_default().push(x); + } + } + } -fn is_plain_number(v: &Value) -> bool { - v.is_number() -} + let mut fields: Vec = present + .into_iter() + .map(|(key, present)| { + let unique_count = uniques.get(&key).map_or(0, BTreeSet::len); + JsonFieldAnalysis { + key: key.clone(), + present, + types: types.remove(&key).unwrap_or_default(), + unique_count, + constant: unique_count == 1 && present == rows.len(), + sparse: present <= sparse_threshold(rows.len()), + numeric: numeric_values.get(&key).map(|values| numeric_stats(values)), + } + }) + .collect(); -fn compact_json(v: &Value) -> String { - serde_json::to_string(v).unwrap_or_default() + fields.sort_by(|a, b| b.present.cmp(&a.present).then_with(|| a.key.cmp(&b.key))); + let columns = fields + .iter() + .map(|field| field.key.clone()) + .collect::>(); + let estimated_table_bytes = rows + .iter() + .map(|row| { + row.values + .values() + .map(render_cell) + .map(|s| s.len() + 3) + .sum::() + }) + .sum::() + + columns.iter().map(String::len).sum::(); + + JsonAnalysis { + row_count: rows.len(), + columns, + fields, + estimated_table_bytes, + } } -/// Format a float without a trailing `.0` for whole numbers. -fn fmt_num(x: f64) -> String { - if x.fract() == 0.0 && x.abs() < 1e15 { - format!("{}", x as i64) - } else { - let s = format!("{x:.3}"); - s.trim_end_matches('0').trim_end_matches('.').to_string() +fn value_type(value: &Value) -> &'static str { + match value { + Value::Null => "null", + Value::Bool(_) => "bool", + Value::Number(_) => "number", + Value::String(_) => "string", + Value::Array(_) => "array", + Value::Object(_) => "object", } } -/// Render a single table cell. Scalars print bare-ish; nested values stay as -/// compact JSON so the table stays lossless (before any truncation). -fn render_cell(v: &Value) -> String { - match v { - Value::String(s) if !s.contains('|') && !s.contains('\n') => escape_markdown_cell(s), - Value::Bool(b) => b.to_string(), - Value::Number(n) => n.to_string(), - other => escape_markdown_cell(&serde_json::to_string(other).unwrap_or_default()), +fn numeric_stats(values: &[f64]) -> JsonNumericStats { + let min = values.iter().copied().fold(f64::INFINITY, f64::min); + let max = values.iter().copied().fold(f64::NEG_INFINITY, f64::max); + let mean = values.iter().sum::() / values.len() as f64; + let variance = values.iter().map(|x| (x - mean).powi(2)).sum::() / values.len() as f64; + JsonNumericStats { + min, + max, + mean, + stddev: variance.sqrt(), } } -/// Truncate an already-escaped cell to [`CELL_MAX`] chars, avoiding a dangling -/// escape backslash. Returns `(cell, was_truncated)`. -fn truncate_cell(s: &str) -> (String, bool) { - if s.chars().count() <= CELL_MAX { - return (s.to_string(), false); - } - let mut cut: String = s.chars().take(CELL_MAX).collect(); - while cut.ends_with('\\') { - cut.pop(); - } - cut.push('…'); - (cut, true) +fn sparse_threshold(row_count: usize) -> usize { + 3.max(row_count / 10) } -/// Truncate a plain (unescaped) string for preamble rendering. -fn truncate_plain(s: &str, max: usize) -> String { - if s.chars().count() <= max { - s.to_string() - } else { - let mut cut: String = s.chars().take(max).collect(); - cut.push('…'); - cut - } +fn dynamic_head_rows(row_count: usize) -> usize { + HEAD_ROWS.min((row_count / 4).max(5)) } -fn render_markdown_row(cells: &[String]) -> String { - format!("| {} |", cells.join(" | ")) +fn dynamic_tail_rows(row_count: usize) -> usize { + TAIL_ROWS.min((row_count / 8).max(3)) } -fn render_markdown_separator(width: usize) -> String { - let cols = vec!["---"; width]; - format!("| {} |", cols.join(" | ")) +fn row_text(row: &FlatRow) -> String { + row.values + .iter() + .map(|(key, value)| format!("{key}={}", render_cell(value))) + .collect::>() + .join(" ") } -fn escape_markdown_cell(cell: &str) -> String { - cell.replace('\\', "\\\\").replace('|', "\\|") +/// Render a single cell. Scalars print bare-ish; nested values stay as compact +/// JSON so the table remains lossless. +fn render_cell(v: &Value) -> String { + match v { + Value::String(s) if !s.contains('|') && !s.contains('\n') => s.clone(), + Value::Bool(b) => b.to_string(), + Value::Number(n) => n.to_string(), + other => serde_json::to_string(other).unwrap_or_default(), + } } #[cfg(test)] mod tests { use super::*; + use crate::cache::{CcrStore, MemoryCcrStore}; #[test] - fn crushes_uniform_array_hoists_constants() { - // `status` and `owner` are identical across all rows → hoisted to a - // preamble note and dropped from the table (feature 3). + fn crushes_uniform_array() { let mut rows = Vec::new(); - for i in 0..80 { + for i in 0..20 { rows.push(format!( r#"{{"id":{i},"name":"item number {i}","status":"active","owner":"team-alpha"}}"# )); } let input = format!("[{}]", rows.join(",")); - let out = compress(&input, false, None).expect("compresses").text; - assert!(out.contains("[all rows:"), "{out}"); - assert!(out.contains(r#"status="active""#), "{out}"); - assert!(out.contains(r#"owner="team-alpha""#), "{out}"); - assert!(out.contains("| id | name |"), "{out}"); - assert_eq!(out.matches("| --- | --- |").count(), 1, "{out}"); + let out = compress(&input).expect("compresses").text; + assert_eq!(out.matches("status").count(), 1, "{out}"); assert!(out.contains("item number 7")); assert!(out.len() < input.len(), "expected shrink"); } + #[test] + fn smartcrusher_table_transform_runs_without_ccr() { + let mut rows = Vec::new(); + for i in 0..20 { + rows.push(format!( + r#"{{"id":{i},"name":"item number {i}","status":"active","owner":"team-alpha"}}"# + )); + } + let input = format!("[{}]", rows.join(",")); + let pipeline_input = PipelineInput { + content: &input, + original_content: &input, + content_kind: ContentKind::Json, + original_bytes: input.len(), + }; + let transform = SmartCrusherTableTransform; + + assert!(transform.applies_to(&pipeline_input)); + let out = transform.apply(&pipeline_input).expect("table reformat"); + + assert_eq!(out.kind, CompressorKind::SmartCrusher); + assert!(out.text.contains("item number 7"), "{}", out.text); + assert!(out.text.len() < input.len(), "{}", out.text); + } + + #[test] + fn smartcrusher_rows_transform_requires_retained_ccr() { + let mut rows = Vec::new(); + for i in 0..140 { + let note = if i == 71 { + "contains special needle token" + } else { + "ordinary row" + }; + rows.push(format!( + r#"{{"id":{i},"name":"record {i}","status":"active","note":"{note}"}}"# + )); + } + let input = format!("[{}]", rows.join(",")); + let pipeline_input = PipelineInput { + content: &input, + original_content: &input, + content_kind: ContentKind::Json, + original_bytes: input.len(), + }; + let transform = SmartCrusherRowsTransform::new().with_query("special needle"); + + let rejecting_store = MemoryCcrStore::new(1, 1); + assert!(transform.apply(&pipeline_input, &rejecting_store).is_none()); + + let store = MemoryCcrStore::default(); + let out = transform + .apply(&pipeline_input, &store) + .expect("retained SmartCrusher rows"); + + assert_eq!(out.kind(), CompressorKind::SmartCrusher); + assert!( + out.text().contains("special needle token"), + "{}", + out.text() + ); + assert_eq!(store.get(out.token()).as_deref(), Some(input.as_str())); + } + + #[test] + fn smartcrusher_rows_transform_skips_reformats_and_non_json_input() { + let mut rows = Vec::new(); + for i in 0..20 { + rows.push(format!( + r#"{{"id":{i},"name":"item number {i}","status":"active","owner":"team-alpha"}}"# + )); + } + let small_json = format!("[{}]", rows.join(",")); + let small_input = PipelineInput { + content: &small_json, + original_content: &small_json, + content_kind: ContentKind::Json, + original_bytes: small_json.len(), + }; + let plain_input = PipelineInput { + content: "plain text", + original_content: "plain text", + content_kind: ContentKind::PlainText, + original_bytes: "plain text".len(), + }; + let transform = SmartCrusherRowsTransform::new(); + let store = MemoryCcrStore::default(); + + assert!(transform.apply(&small_input, &store).is_none()); + assert_eq!(transform.estimate_bloat(&plain_input), 0.0); + assert!(transform.apply(&plain_input, &store).is_none()); + } + #[test] fn large_array_row_drops_and_is_marked_lossy() { let mut rows = Vec::new(); for i in 0..200 { rows.push(format!( - r#"{{"id":{i},"name":"record number {i}","status":"s{i}","note":"some detail {i}"}}"# + r#"{{"id":{i},"name":"record number {i}","status":"active","note":"some detail {i}"}}"# )); } let input = format!("[{}]", rows.join(",")); - // Row-dropping is the CCR path: it only fires with block tokens on. - let c = compress(&input, true, None).expect("compresses"); + let c = compress(&input).expect("compresses"); assert!(c.lossy, "row-dropped output must be lossy"); assert!(c.text.contains("record number 0"), "{}", c.text); assert!(c.text.contains("record number 199"), "{}", c.text); @@ -1046,418 +1349,518 @@ mod tests { assert!(c.text.len() < input.len()); } - /// Without CCR the same large array is reshaped into a full markdown table: - /// every row survives (lossless), nothing is dropped, and it still shrinks - /// well below the pretty-printed JSON — the "markdown trick". #[test] - fn large_array_without_ccr_is_a_full_lossless_table() { + fn analyzer_reports_constants_sparse_fields_and_numeric_stats() { let mut rows = Vec::new(); - for i in 0..200 { + for i in 0..20 { + let extra = if i == 13 { + r#","debug":"only here""# + } else { + "" + }; rows.push(format!( - r#"{{"id":{i},"name":"record number {i}","status":"s{i}","note":"some detail {i}"}}"# + r#"{{"id":{i},"status":"active","latency":{}{extra}}}"#, + 10 + i )); } let input = format!("[{}]", rows.join(",")); - let c = compress(&input, false, None).expect("compresses"); - assert!(!c.lossy, "full table is a faithful reshape: {}", c.text); - assert!(!c.text.contains("omitted"), "no rows dropped: {}", c.text); - assert!(c.text.contains("faithful reshape"), "{}", c.text); - // Every row's identifying values survive, including the middle ones a - // row-drop would have sampled away. - for i in [0usize, 75, 137, 199] { - assert!( - c.text.contains(&format!("record number {i}")), - "row {i} kept: {}", - c.text - ); - } - assert!(c.text.len() < input.len(), "still shrinks vs pretty JSON"); + + let analysis = analyze(&input).expect("analysis"); + let status = analysis + .fields + .iter() + .find(|field| field.key == "status") + .expect("status field"); + let debug = analysis + .fields + .iter() + .find(|field| field.key == "debug") + .expect("debug field"); + let latency = analysis + .fields + .iter() + .find(|field| field.key == "latency") + .expect("latency field"); + + assert!(status.constant); + assert_eq!(status.unique_ratio(), 1.0 / 20.0); + assert!(debug.sparse); + assert_eq!(debug.present, 1); + assert_eq!(debug.unique_ratio(), 1.0); + assert_eq!(latency.numeric.expect("numeric").min, 10.0); + assert!(latency.unique_ratio() > 0.9); + assert!(analysis.estimated_table_bytes < input.len()); + assert!(analysis.estimated_reduction_bytes(input.len()) > 0); + assert!(analysis.estimated_reduction_ratio(input.len()) > 0.0); } #[test] - fn keeps_error_row_in_dropped_middle() { + fn constant_columns_are_hoisted_out_of_rows() { let mut rows = Vec::new(); - for i in 0..120 { - let status = if i == 75 { "error: timeout" } else { "ok" }; + for i in 0..20 { rows.push(format!( - r#"{{"id":{i},"name":"job {i}","status":"{status}","note":"detail {i}"}}"# + r#"{{"id":{i},"status":"active","region":"us-east","latency":{}}}"#, + 10 + i + )); + } + let input = format!("[{}]", rows.join(",")); + + let c = compress(&input).expect("compresses"); + + assert!(!c.lossy); + assert!(c.text.contains("constants: region=us-east | status=active")); + assert!(c.text.contains("id | latency"), "{}", c.text); + assert!(!c.text.contains("id | latency | region"), "{}", c.text); + assert_eq!(c.text.matches("active").count(), 1, "{}", c.text); + assert_eq!(c.text.matches("us-east").count(), 1, "{}", c.text); + } + + #[test] + fn query_anchor_row_survives_dropped_middle() { + let mut rows = Vec::new(); + for i in 0..140 { + let note = if i == 71 { + "contains special needle token" + } else { + "ordinary row" + }; + rows.push(format!( + r#"{{"id":{i},"name":"record {i}","status":"active","note":"{note}"}}"# )); } let input = format!("[{}]", rows.join(",")); - let c = compress(&input, true, None).expect("compresses"); + + let c = compress_with_query(&input, Some("special needle")).expect("compresses"); + assert!(c.lossy); - assert!( - c.text.contains("job 75"), - "error row must survive:\n{}", - c.text - ); - assert!(c.text.contains("error: timeout")); + assert!(c.text.contains("special needle token"), "{}", c.text); } #[test] - fn keeps_nested_error_row() { - // Feature 2: error text lives inside a nested object, not a top-level - // string. The row must still be kept. + fn latest_query_extends_tail_anchor_window() { let mut rows = Vec::new(); - for i in 0..120 { - let detail = if i == 63 { - r#"{"code":"CONFLICT_PACKAGES","message":"resolution failed"}"# + for i in 0..140 { + let note = if i == 124 { + "latest positional context".to_string() } else { - r#"{"code":"OK","message":"done"}"# + format!("ordinary row {i}") }; rows.push(format!( - r#"{{"id":{i},"name":"pkg {i}","result":{detail}}}"# + r#"{{"id":{i},"name":"record {i}","status":"active","note":"{note}"}}"# )); } let input = format!("[{}]", rows.join(",")); - let c = compress(&input, false, None).expect("compresses"); - assert!( - c.text.contains("pkg 63") || c.text.contains("CONFLICT_PACKAGES"), - "nested error row must survive:\n{}", - c.text - ); + + let c = compress_with_query(&input, Some("latest records")).expect("compresses"); + + assert!(c.lossy); + assert!(c.text.contains("latest positional context"), "{}", c.text); } #[test] - fn keeps_numeric_outlier_row() { + fn oldest_query_extends_head_anchor_window() { let mut rows = Vec::new(); - for i in 0..120 { - let latency = if i == 88 { 9999 } else { 10 + (i % 3) }; + for i in 0..140 { + let note = if i == 24 { + "oldest positional context".to_string() + } else { + format!("ordinary row {i}") + }; rows.push(format!( - r#"{{"id":{i},"endpoint":"/api/{i}","latency_ms":{latency},"region":"r{i}"}}"# + r#"{{"id":{i},"name":"record {i}","status":"active","note":"{note}"}}"# )); } let input = format!("[{}]", rows.join(",")); - let c = compress(&input, false, None).expect("compresses"); - assert!( - c.text.contains("9999"), - "outlier row must survive:\n{}", - c.text - ); + + let c = compress_with_query(&input, Some("oldest records")).expect("compresses"); + + assert!(c.lossy); + assert!(c.text.contains("oldest positional context"), "{}", c.text); } #[test] - fn keeps_pareto_rare_status_row() { - // Feature 5: `status` is 99% "ok"; the single "degraded" row is a - // categorical rarity that must be kept even in the drop window. + fn sparse_structural_row_survives_dropped_middle() { let mut rows = Vec::new(); - for i in 0..120 { - let status = if i == 70 { "degraded" } else { "ok" }; + for i in 0..140 { + let extra = if i == 72 { + r#","diagnostic":"rare sparse field""# + } else { + "" + }; rows.push(format!( - r#"{{"id":{i},"endpoint":"/api/{i}","status":"{status}"}}"# + r#"{{"id":{i},"name":"record {i}","status":"active"{extra}}}"# )); } let input = format!("[{}]", rows.join(",")); - let c = compress(&input, false, None).expect("compresses"); - assert!( - c.text.contains("degraded"), - "rare categorical value must survive:\n{}", - c.text - ); + + let c = compress(&input).expect("compresses"); + + assert!(c.lossy); + assert!(c.text.contains("rare sparse field"), "{}", c.text); } #[test] - fn keeps_rare_structural_field_row() { - // Feature 6: only one row carries `escalation` → kept. + fn discriminator_bucket_row_survives_dropped_middle() { let mut rows = Vec::new(); - for i in 0..120 { - if i == 55 { + for i in 0..140 { + let kind = if i == 72 { + "audit" + } else if i % 2 == 0 { + "service" + } else { + "worker" + }; + let message = if i == 72 { + "audit bucket unique payload" + } else { + "ordinary payload" + }; + rows.push(format!( + r#"{{"id":{i},"kind":"{kind}","status":"active","message":"{message}"}}"# + )); + } + let input = format!("[{}]", rows.join(",")); + + let c = compress(&input).expect("compresses"); + + assert!(c.lossy); + assert!(c.text.contains("audit"), "{}", c.text); + assert!(c.text.contains("audit bucket unique payload"), "{}", c.text); + } + + #[test] + fn duplicate_cluster_representative_survives_dropped_middle() { + let mut rows = Vec::new(); + for i in 0..140 { + if (72..=76).contains(&i) { + rows.push( + r#"{"id":"retry-batch-17","status":"retry","message":"duplicate cluster payload"}"# + .to_string(), + ); + } else { rows.push(format!( - r#"{{"id":{i},"name":"t{i}","escalation":"paged oncall"}}"# + r#"{{"id":"job-{i}","status":"ok","message":"ordinary payload {i}"}}"# )); - } else { - rows.push(format!(r#"{{"id":{i},"name":"t{i}"}}"#)); } } let input = format!("[{}]", rows.join(",")); - let c = compress(&input, false, None).expect("compresses"); + + let c = compress(&input).expect("compresses"); + + assert!(c.lossy); + assert!(c.text.contains("retry-batch-17"), "{}", c.text); + assert!(c.text.contains("duplicate cluster payload"), "{}", c.text); + } + + #[test] + fn near_duplicate_cluster_representative_survives_dropped_middle() { + let mut rows = Vec::new(); + for i in 0..140 { + let message = if (72..=76).contains(&i) { + "retry backlog service failed" + } else { + "routine healthy service stable" + }; + rows.push(format!( + r#"{{"id":"job-{i}","status":"active","message":"{message}"}}"# + )); + } + let input = format!("[{}]", rows.join(",")); + + let c = compress(&input).expect("compresses"); + + assert!(c.lossy); + assert!(c.text.contains("job-72"), "{}", c.text); assert!( - c.text.contains("paged oncall") || c.text.contains("t55"), - "rare-field row must survive:\n{}", + c.text.contains("retry backlog service failed"), + "{}", c.text ); } #[test] - fn keeps_query_anchor_row() { - // Feature 8: a distinctive id in the query pins the matching row. + fn spread_anchor_row_survives_dropped_middle() { let mut rows = Vec::new(); - for i in 0..120 { + for i in 0..140 { + let message = if i == 94 { + "deterministic spread anchor payload".to_string() + } else { + format!("ordinary payload {i}") + }; rows.push(format!( - r#"{{"id":{i},"name":"widget {i}","ticket":{}}}"#, - 5000 + i + r#"{{"id":{i},"name":"record {i}","status":"active","message":"{message}"}}"# )); } let input = format!("[{}]", rows.join(",")); - let c = - compress(&input, false, Some("investigate ticket 5087 failure")).expect("compresses"); + + let c = compress(&input).expect("compresses"); + + assert!(c.lossy); assert!( - c.text.contains("widget 87"), - "anchored row must survive:\n{}", + c.text.contains("deterministic spread anchor payload"), + "{}", c.text ); } #[test] - fn flattens_uniform_object_column() { - // Feature 4: the `function` column is a uniform object; name/description - // are promoted to columns instead of a per-row JSON blob. + fn adaptive_saturation_extends_spread_anchors() { let mut rows = Vec::new(); - for i in 0..30 { + for i in 0..140 { + let word = alpha_word(i); rows.push(format!( - r#"{{"type":"function","function":{{"name":"TOOL_{i}","description":"does thing {i}","parameters":{{"type":"object"}}}}}}"# + r#"{{"id":{i},"status":"active","message":"topic {word} marker alpha beta"}}"# )); } let input = format!("[{}]", rows.join(",")); - let out = compress(&input, false, None).expect("compresses").text; - assert!(out.contains("function.name"), "{out}"); - assert!(out.contains("function.description"), "{out}"); - assert!(out.contains("TOOL_3"), "{out}"); - // `type` is constant → hoisted, not a column. - assert!(out.contains(r#"type="function""#), "{out}"); + let expected = alpha_word(108); + + let c = compress(&input).expect("compresses"); + + assert!(c.lossy); + assert!(c.text.contains(&expected), "{}", c.text); } #[test] - fn tables_nested_array_with_path_prefix() { - // Feature 1: top-level object whose largest uniform-object array is - // nested under `methods`. - let mut methods = Vec::new(); - for i in 0..69 { - methods.push(format!( - r#"{{"method":"core.m{i}","namespace":"core","description":"method {i}"}}"# + fn early_saturation_keeps_base_spread_anchors() { + let mut rows = Vec::new(); + for i in 0..140 { + let message = if i < 70 { + format!("topic {} marker alpha beta", alpha_word(i)) + } else { + "topic repeated marker alpha beta".to_string() + }; + rows.push(format!( + r#"{{"id":"job-{i}","status":"active","message":"{message}"}}"# )); } - let input = format!(r#"{{"version":"1.0","methods":[{}]}}"#, methods.join(",")); - let out = compress(&input, false, None).expect("compresses").text; - assert!(out.contains("[json table: methods —"), "{out}"); - assert!(out.contains("69 rows"), "{out}"); - // Sibling scalar rendered as preamble. - assert!(out.contains("version: 1.0"), "{out}"); - } + let input = format!("[{}]", rows.join(",")); + + let c = compress(&input).expect("compresses"); - #[test] - fn crushes_number_array() { - // Feature 7: plain-number array → distributional stats. - let nums: Vec = (0..200).map(|i| (i * 3).to_string()).collect(); - let input = format!("[{}]", nums.join(",")); - let c = compress(&input, false, None).expect("compresses"); - assert!(c.text.contains("json number array"), "{}", c.text); - assert!(c.text.contains("min=0"), "{}", c.text); - assert!(c.text.contains("max=597"), "{}", c.text); - assert!(c.text.contains("median="), "{}", c.text); assert!(c.lossy); - assert!(c.text.len() < input.len()); + assert!(c.text.contains("job-94"), "{}", c.text); + } + + fn alpha_word(mut n: usize) -> String { + let mut out = String::from("word"); + loop { + out.push((b'a' + (n % 26) as u8) as char); + n /= 26; + if n == 0 { + break; + } + } + out } #[test] - fn crushes_string_array_with_dedupe() { - // Feature 7: plain-string array → deduped counts. - let mut vals = Vec::new(); - for _ in 0..50 { - vals.push("\"apple\"".to_string()); - } - for _ in 0..30 { - vals.push("\"banana\"".to_string()); - } - let input = format!("[{}]", vals.join(",")); - let c = compress(&input, false, None).expect("compresses"); - assert!(c.text.contains("json string array"), "{}", c.text); - assert!(c.text.contains("2 unique"), "{}", c.text); - assert!(c.text.contains("×50"), "{}", c.text); - assert!(c.text.len() < input.len()); + fn numeric_change_point_row_survives_dropped_middle() { + let mut rows = Vec::new(); + for i in 0..140 { + let phase = if i < 72 { 10 } else { 30 }; + let message = if i == 72 { + "phase transition payload".to_string() + } else { + format!("ordinary payload {i}") + }; + rows.push(format!( + r#"{{"id":{i},"phase_score":{phase},"status":"active","message":"{message}"}}"# + )); + } + let input = format!("[{}]", rows.join(",")); + + let c = compress(&input).expect("compresses"); + + assert!(c.lossy); + assert!(c.text.contains("phase transition payload"), "{}", c.text); } #[test] - fn non_crushable_returns_none() { - assert!(compress(r#"{"a":1}"#, false, None).is_none()); - assert!(compress("[1,2,3]", false, None).is_none()); - assert!(compress(r#"[{"a":1}]"#, false, None).is_none()); + fn information_dense_row_survives_dropped_middle() { + let mut rows = Vec::new(); + for i in 0..140 { + let message = if i == 72 { + "trace alpha beta gamma delta epsilon zeta eta theta iota kappa lambda".to_string() + } else { + format!("ordinary payload {i}") + }; + rows.push(format!( + r#"{{"id":{i},"status":"active","message":"{message}"}}"# + )); + } + let input = format!("[{}]", rows.join(",")); + + let c = compress(&input).expect("compresses"); + + assert!(c.lossy); + assert!(c.text.contains("trace alpha beta gamma"), "{}", c.text); } #[test] - fn markdown_table_cells_escape_pipes() { - // `meta` has only a non-informative `note` sub-key → NOT flattened, so - // the nested-JSON escaping path stays exercised. - let mut rows = vec![r#"{"id":0,"text":"alpha | beta","meta":{"note":"x|y"}}"#.to_string()]; - for i in 1..80 { + fn nested_objects_flatten_to_dotted_columns() { + let mut rows = Vec::new(); + for i in 0..20 { rows.push(format!( - r#"{{"id":{i},"text":"record {i} with repeated detail","meta":{{"note":"z"}}}}"# + r#"{{"id":{i},"user":{{"name":"user {i}","team":"core"}},"status":"active"}}"# )); } let input = format!("[{}]", rows.join(",")); - let out = compress(&input, false, None).expect("compresses").text; - assert!(out.contains("| id | meta | text |"), "{out}"); - assert!(out.contains("alpha \\| beta"), "{out}"); - assert!(out.contains(r#"{"note":"x\|y"}"#), "{out}"); + + let c = compress(&input).expect("compresses"); + + assert!(c.text.contains("user.name"), "{}", c.text); + assert!(c.text.contains("user.team"), "{}", c.text); + assert!(c.text.contains("user 7"), "{}", c.text); } #[test] - fn falls_back_to_minified_json_when_table_is_larger() { - let input = r#"[ - {"a": 1, "b": 2}, - {"a": 3, "b": 4}, - {"a": 5, "b": 6} - ]"#; - let out = compress(input, false, None).expect("minifies").text; - assert_eq!(out, r#"[{"a":1,"b":2},{"a":3,"b":4},{"a":5,"b":6}]"#); + fn stringified_json_objects_flatten_to_dotted_columns() { + let mut rows = Vec::new(); + for i in 0..20 { + let metadata = serde_json::json!({ + "owner": format!("team-{i}"), + "flags": { "retry": i % 2 == 0 } + }) + .to_string() + .replace('"', "\\\""); + rows.push(format!( + r#"{{"id":{i},"metadata":"{metadata}","status":"active"}}"# + )); + } + let input = format!("[{}]", rows.join(",")); + + let c = compress(&input).expect("compresses"); + + assert!(c.text.contains("metadata.owner"), "{}", c.text); + assert!(c.text.contains("metadata.flags.retry"), "{}", c.text); + assert!(c.text.contains("team-7"), "{}", c.text); } #[test] - fn minified_json_wins_when_table_is_smaller_than_original_but_larger_than_minified() { + fn stringified_json_parsing_leaves_scalar_and_invalid_strings_opaque() { let input = r#"[ - {"enabled": true, "id": 1}, - {"enabled": false, "id": 2}, - {"enabled": true, "id": 3}, - {"enabled": false, "id": 4}, - {"enabled": true, "id": 5} + {"id":1,"payload":"001","note":"{not json}"}, + {"id":2,"payload":"002","note":"[also not json]"}, + {"id":3,"payload":"003","note":"plain"} ]"#; - let out = compress(input, false, None).expect("minifies").text; - assert_eq!( - out, - r#"[{"enabled":true,"id":1},{"enabled":false,"id":2},{"enabled":true,"id":3},{"enabled":false,"id":4},{"enabled":true,"id":5}]"# - ); + + let c = compress(input).expect("compresses"); + + assert!(c.text.contains("payload"), "{}", c.text); + assert!(c.text.contains("001"), "{}", c.text); + assert!(c.text.contains("{not json}"), "{}", c.text); + assert!(!c.text.contains("note."), "{}", c.text); } #[test] - fn omitted_rows_token_retrieves_json_slice() { - use crate::cache; - let mut items = Vec::new(); - for i in 0..120 { - items.push(format!(r#"{{"id":{i},"name":"row_{i}","status":"st{i}"}}"#)); - } - let input = format!("[{}]", items.join(",")); - let out = compress(&input, true, None).expect("compresses").text; - let tokens = cache::parse_markers(&out); - assert!(!tokens.is_empty(), "row gaps should carry tokens: {out}"); - let block = cache::retrieve(&tokens[0]).expect("stored"); - let parsed: serde_json::Value = serde_json::from_str(&block).expect("valid JSON slice"); - let arr = parsed.as_array().expect("array slice"); - assert!(!arr.is_empty()); - // The slice starts exactly where the adaptive head window ends, and - // none of its rows also appear in the rendered table. - let first_id = arr[0]["id"].as_u64().expect("id") as usize; - assert!(first_id > 0, "head must keep at least one row"); - assert_eq!(arr[0]["name"], format!("row_{first_id}")); - assert!( - out.contains(&format!("row_{}", first_id - 1)), - "row before the gap must be shown:\n{out}" - ); - for row in arr { - let name = row["name"].as_str().expect("name"); - assert!( - !out.contains(&format!("| {name} |")), - "omitted row {name} must not also be rendered:\n{out}" - ); + fn heterogeneous_shapes_render_smaller_bucket_tables() { + let mut rows = Vec::new(); + for i in 0..12 { + rows.push(format!( + r#"{{"event":"login","user_id":"user-{i}","ip":"10.0.0.{i}","success":true}}"# + )); + rows.push(format!( + r#"{{"event":"deploy","service":"api-{i}","version":"2026.{i}.0","region":"us-east"}}"# + )); + rows.push(format!( + r#"{{"event":"metric","name":"cpu-{i}","value":{},"unit":"pct"}}"#, + 40 + i + )); } - } + let input = format!("[{}]", rows.join(",")); - /// Count rendered table body rows (data rows, excluding header/separator). - fn body_row_count(text: &str) -> usize { - text.lines() - .filter(|l| l.starts_with("| ") && !l.starts_with("| ---")) - .count() - .saturating_sub(1) // header + let c = compress(&input).expect("compresses"); + + assert!(!c.lossy); + assert!(c.text.contains("[json bucketed tables:"), "{}", c.text); + assert!(c.text.contains("[bucket 1:"), "{}", c.text); + assert!(c.text.contains("[bucket 2:"), "{}", c.text); + assert!(c.text.contains("[bucket 3:"), "{}", c.text); + assert!(c.text.contains("constants: event=login | success=true")); + assert!(c.text.contains("constants: event=deploy | region=us-east")); + assert!(c.text.contains("constants: event=metric | unit=pct")); + assert!(c.text.contains("__row | ip | user_id")); + assert!(c.text.contains("__row | service | version")); + assert!(c.text.contains("__row | name | value")); + assert!(c.text.contains("31 | api-10 | 2026.10.0")); } #[test] - fn redundant_rows_collapse_toward_floor() { - // 100 near-identical rows: adaptive selection should keep close to - // MIN_KEPT_ROWS, far below the fixed HEAD_ROWS+TAIL_ROWS budget. + fn lossy_heterogeneous_bucket_marks_omissions_and_recovery() { let mut rows = Vec::new(); - for i in 0..100 { + for i in 0..90 { + rows.push(format!( + r#"{{"event":"login","user_id":"user-{i}","ip":"10.0.0.{i}","success":true}}"# + )); + } + for i in 0..90 { rows.push(format!( - r#"{{"id":{i},"msg":"connection timeout retrying request to upstream host","status":"ok"}}"# + r#"{{"event":"metric","name":"cpu-{i}","value":{},"unit":"pct"}}"#, + 40 + i )); } let input = format!("[{}]", rows.join(",")); - let c = compress(&input, true, None).expect("compresses"); - let kept = body_row_count(&c.text); - assert!( - kept >= MIN_KEPT_ROWS, - "never fewer than the floor, got {kept}:\n{}", - c.text - ); - assert!( - kept <= MIN_KEPT_ROWS + 4, - "redundant rows must stay near the floor, got {kept}:\n{}", - c.text - ); - assert!(c.text.contains("omitted"), "{}", c.text); + + let value: Value = serde_json::from_str(&input).unwrap(); + let flat_rows = flatten_rows(value.as_array().unwrap()).unwrap(); + let c = compress_shape_buckets(&input, &flat_rows, None).expect("bucketed"); + + assert!(c.lossy); + assert!(c.text.contains("[json bucketed tables:"), "{}", c.text); + assert!(c.text.contains("exact original via retrieve footer")); + assert!(c.text.contains("row(s) omitted in bucket"), "{}", c.text); + assert!(c.text.contains("constants: event=login | success=true")); + assert!(c.text.contains("0 | 10.0.0.0 | user-0"), "{}", c.text); + assert!(c.text.contains("constants: event=metric | unit=pct")); + assert!(c.text.contains("179 | cpu-89 | 129"), "{}", c.text); } #[test] - fn diverse_rows_keep_full_budget() { - // 100 fully-diverse rows: adaptive selection should keep the full - // HEAD_ROWS + TAIL_ROWS budget. - let words = [ - "database", - "login", - "cache", - "worker", - "handshake", - "latency", - "rollout", - "webhook", - "resolver", - "kernel", - "exporter", - "backlog", - "renewal", - "election", - "limiter", - "index", - "cookie", - "saturated", - "multipart", - "deadline", - ]; + fn keeps_error_row_in_dropped_middle() { + // A homogeneous array with a single error row buried in the middle: the + // SmartCrusher must keep that row even though it's in the drop window. let mut rows = Vec::new(); - for i in 0..100 { + for i in 0..120 { + let status = if i == 75 { "error: timeout" } else { "ok" }; rows.push(format!( - r#"{{"event":"{} {} {} incident {i}","code":"{}{}","weight":{}}}"#, - words[i % 20], - words[(i * 7 + 3) % 20], - words[(i * 13 + 11) % 20], - words[(i * 3 + 5) % 20], - i * 977, - i * i + 17, + r#"{{"id":{i},"name":"job {i}","status":"{status}","note":"detail {i}"}}"# )); } let input = format!("[{}]", rows.join(",")); - let c = compress(&input, true, None).expect("compresses"); - let kept = body_row_count(&c.text); + let c = compress(&input).expect("compresses"); + assert!(c.lossy); assert!( - kept >= HEAD_ROWS + TAIL_ROWS, - "diverse rows should keep the full budget, got {kept}:\n{}", + c.text.contains("job 75"), + "error row must survive:\n{}", c.text ); + assert!(c.text.contains("error: timeout")); } #[test] - fn must_keep_rows_are_additive_over_adaptive_window() { - // Redundant array (small adaptive window) with an error row buried in - // the dropped middle: the error row is kept ON TOP of the window. + fn keeps_numeric_outlier_row() { let mut rows = Vec::new(); - for i in 0..100 { - let msg = if i == 60 { - "error: upstream exploded" - } else { - "connection timeout retrying request to upstream host" - }; - rows.push(format!(r#"{{"id":{i},"msg":"{msg}","status":"ok"}}"#)); + for i in 0..120 { + // Most latencies ~10ms; row 88 is a 9999ms outlier. + let latency = if i == 88 { 9999 } else { 10 + (i % 3) }; + rows.push(format!( + r#"{{"id":{i},"endpoint":"/api/{i}","latency_ms":{latency},"region":"us"}}"# + )); } let input = format!("[{}]", rows.join(",")); - let c = compress(&input, true, None).expect("compresses"); + let c = compress(&input).expect("compresses"); assert!( - c.text.contains("error: upstream exploded"), - "must-keep error row must survive the adaptive window:\n{}", - c.text - ); - let kept = body_row_count(&c.text); - assert!( - kept <= MIN_KEPT_ROWS + 5, - "window should stay small even with the extra must-keep, got {kept}:\n{}", + c.text.contains("9999"), + "outlier row must survive:\n{}", c.text ); } + + #[test] + fn non_array_returns_none() { + assert!(compress(r#"{"a":1}"#).is_none()); + assert!(compress("[1,2,3]").is_none()); + assert!(compress(r#"[{"a":1}]"#).is_none()); + } } diff --git a/src/compressors/log.rs b/src/compressors/log.rs index acc252f..02099bf 100644 --- a/src/compressors/log.rs +++ b/src/compressors/log.rs @@ -15,40 +15,82 @@ //! listing, CSV, generated data) and must not be head/tail truncated. use async_trait::async_trait; -use std::collections::{BTreeSet, HashMap}; +use once_cell::sync::Lazy; +use std::collections::HashSet; use std::fmt::Write as _; -use super::log_template; +use super::Compressor; use super::signals::{Severity, severity}; -use super::{BLOCK_NOTE, Compressor, block_token}; +use crate::cache::CcrStore; +use crate::pipeline::{ + OffloadOutput, OffloadTransform, PipelineInput, ReformatTransform, TransformOutput, + estimate_bloat, +}; use crate::reduce::reduce_execution_with_rules; -use crate::rules::cached_overlay_rules; +use crate::rules::load_builtin_rules; use crate::types::{ - CompressInput, CompressOptions, CompressOutput, CompressorKind, ReduceOptions, - ToolExecutionInput, + CompiledRule, CompressInput, CompressOptions, CompressOutput, CompressorKind, ContentKind, + ReduceOptions, ToolExecutionInput, }; -/// Safety ceiling on distinct error templates kept (all distinct templates are -/// kept below this; the cap only guards a pathological blob). -pub const MAX_ERROR_TEMPLATES: usize = 40; -/// Distinct warning templates kept. -pub const MAX_WARNING_TEMPLATES: usize = 10; +pub const MAX_ERRORS: usize = 10; +pub const MAX_WARNINGS: usize = 5; pub const MAX_STACK_TRACES: usize = 3; pub const STACK_TRACE_MAX_LINES: usize = 20; pub const MAX_TOTAL_LINES: usize = 100; +pub const TEMPLATE_MIN_RUN: usize = 8; +pub const TEMPLATE_MIN_CONSTANT_TOKENS: usize = 3; -/// Lines of causal context kept before/after each distinct error exemplar. -const CONTEXT_BEFORE: usize = 2; -const CONTEXT_AFTER: usize = 2; -/// Gaps of at most this many lines are emitted verbatim rather than replaced by -/// a marker + CCR token (the marker would outweigh the omitted text). -const MIN_GAP: usize = 2; -/// Only mine a dropped region for template summaries when it is at least this -/// many lines — small gaps aren't worth a summary line. -const MIN_REGION_FOR_TEMPLATES: usize = 6; +static BUILTIN_RULES: Lazy> = Lazy::new(load_builtin_rules); pub struct LogCompressor; +/// Typed reformat transform for repetitive non-command logs. +pub struct LogTemplateTransform; + +/// Typed offload transform for signal-preserving non-command logs. +pub struct SignalLogTransform; + +impl ReformatTransform for LogTemplateTransform { + fn name(&self) -> &'static str { + "log_templates" + } + + fn applies_to(&self, input: &PipelineInput<'_>) -> bool { + input.content_kind == ContentKind::Log + } + + fn apply(&self, input: &PipelineInput<'_>) -> Option { + let output = compress_templates(input.content)?; + Some(TransformOutput::new(output.text, CompressorKind::Log)) + } +} + +impl OffloadTransform for SignalLogTransform { + fn name(&self) -> &'static str { + "signal_log" + } + + fn estimate_bloat(&self, input: &PipelineInput<'_>) -> f32 { + if input.content_kind != ContentKind::Log { + return 0.0; + } + f32::from(estimate_bloat(input.content, input.content_kind).score) / 100.0 + } + + fn apply(&self, input: &PipelineInput<'_>, store: &dyn CcrStore) -> Option { + if input.content_kind != ContentKind::Log { + return None; + } + let compacted = compress_signal(input.content)?; + OffloadOutput::from_retained_put( + compacted.text, + CompressorKind::Log, + store.put(input.original_content), + ) + } +} + #[async_trait] impl Compressor for LogCompressor { fn kind(&self) -> CompressorKind { @@ -62,75 +104,14 @@ impl Compressor for LogCompressor { ) -> Option { let has_command = input.command.is_some() || input.argv.as_ref().is_some_and(|a| !a.is_empty()); - // The drop-based view wins whenever it will actually be emitted — with - // CCR every dropped line is recoverable, and a host that opts into - // `lossy_without_ccr` accepts marked-but-unrecoverable output. When - // neither holds (the strict no-CCR default) that view would be declined - // by the router as unrecoverable, so fall back to the one lossless log - // transform: collapse runs of byte-identical lines. - let lossy_ok = opts.ccr_enabled || opts.lossy_without_ccr; - if lossy_ok { - let lossy = if has_command { - compress_command(input, opts) - } else { - compress_signal(input.content, opts.ccr_enabled) - }; - lossy.or_else(|| lossless_run_length(input.content)) + if has_command { + compress_command(input, opts) } else { - lossless_run_length(input.content) + compress_templates(input.content).or_else(|| compress_signal(input.content)) } } } -/// Collapse runs of *byte-identical consecutive* lines into a single line plus -/// a `[×N]` repeat count. Lossless: the exact line text and its repeat count -/// both survive and order is preserved (only adjacent duplicates merge), so the -/// reader can reconstruct the run — no line is dropped. Each run only collapses -/// when doing so actually saves bytes; returns `None` when nothing collapses or -/// the result wouldn't shrink. This is the compressor's no-CCR fallback: unlike -/// the signal/rule paths it drops no information, so it ships without a cache. -fn lossless_run_length(content: &str) -> Option { - // split('\n') + join('\n') round-trips the exact line/newline structure, - // including a trailing newline (which yields a final empty element). - let lines: Vec<&str> = content.split('\n').collect(); - let mut out: Vec = Vec::with_capacity(lines.len()); - let mut collapsed = false; - let mut i = 0; - while i < lines.len() { - let line = lines[i]; - let mut j = i + 1; - while j < lines.len() && lines[j] == line { - j += 1; - } - let run = j - i; - let merged = format!("{line} [×{run}]"); - // Only merge when it beats re-emitting the run verbatim (run copies of - // the line, one newline each). - if run >= 2 && merged.len() + 1 < (line.len() + 1) * run { - out.push(merged); - collapsed = true; - } else { - for _ in 0..run { - out.push(line.to_string()); - } - } - i = j; - } - if !collapsed { - return None; - } - let text = out.join("\n"); - if text.len() >= content.len() { - return None; - } - log::debug!( - "[tinyjuice][log] lossless run-length {} -> {} bytes", - content.len(), - text.len() - ); - Some(CompressOutput::reformatted(text, CompressorKind::Log)) -} - /// Command output → run the rule engine, tagged as [`CompressorKind::Log`]. fn compress_command(input: &CompressInput<'_>, opts: &CompressOptions) -> Option { run_rule_engine(input, opts, CompressorKind::Log) @@ -169,10 +150,7 @@ fn run_rule_engine( max_inline_chars: opts.max_inline_chars, ..Default::default() }; - // Full three-layer overlay (builtin → user → project) so project rules in - // `/.tinyjuice/rules/` apply to the compression path, as documented. - let rules = cached_overlay_rules(None); - let result = reduce_execution_with_rules(exec, &rules, &reduce_opts); + let result = reduce_execution_with_rules(exec, &BUILTIN_RULES, &reduce_opts); if result.inline_text.len() >= input.content.len() { return None; } @@ -181,7 +159,7 @@ fn run_rule_engine( .matched_reducer .unwrap_or(result.classification.family); log::debug!( - "[tinyjuice][log] command rule={} kind={} {} -> {} bytes", + "[tokenjuice][log] command rule={} kind={} {} -> {} bytes", rule_label, kind.as_str(), input.content.len(), @@ -190,18 +168,14 @@ fn run_rule_engine( Some(CompressOutput::lossy(result.inline_text, kind)) } -/// Signal-based log compression for non-command blobs detected as logs. With -/// `block_tokens`, each omitted gap is offloaded to CCR and its marker carries -/// a token that retrieves exactly those lines. -pub fn compress_signal(content: &str, block_tokens: bool) -> Option { +/// Signal-based log compression for non-command blobs detected as logs. +pub fn compress_signal(content: &str) -> Option { let lines: Vec<&str> = content.lines().collect(); if lines.len() <= MAX_TOTAL_LINES { return None; } - let mut keep: BTreeSet = BTreeSet::new(); - // Per-kept-line suffixes (e.g. `×N` template counts) appended on emit. - let mut annotations: HashMap = HashMap::new(); + let mut keep: std::collections::BTreeSet = std::collections::BTreeSet::new(); for (i, line) in lines.iter().enumerate() { if is_summary_line(line) { @@ -209,40 +183,38 @@ pub fn compress_signal(content: &str, block_tokens: bool) -> Option = lines.iter().map(|l| is_stack_frame(l)).collect(); - - // Errors: keep one exemplar per distinct template with an `×N` count and a - // small window of causal context; keep every distinct template up to a - // safety ceiling (no flat first-few/last-few cap). - collapse_by_template( - &lines, - |i| severity(lines[i]) == Severity::Error && !stack_frame[i], - MAX_ERROR_TEMPLATES, - true, - &mut keep, - &mut annotations, - ); + let error_idx: Vec = lines + .iter() + .enumerate() + .filter(|(_, l)| severity(l) == Severity::Error) + .map(|(i, _)| i) + .collect(); + for &i in select_first_last(&error_idx, MAX_ERRORS).iter() { + keep.insert(i); + } - // Warnings: exemplar + count per distinct template, fewer distinct kept. - collapse_by_template( - &lines, - |i| severity(lines[i]) == Severity::Warning && !stack_frame[i], - MAX_WARNING_TEMPLATES, - false, - &mut keep, - &mut annotations, - ); + let mut seen_warn: HashSet = HashSet::new(); + let mut warn_kept = 0usize; + for (i, line) in lines.iter().enumerate() { + if warn_kept >= MAX_WARNINGS { + break; + } + if severity(line) == Severity::Warning { + let norm = normalize_for_dedupe(line); + if seen_warn.insert(norm) { + keep.insert(i); + warn_kept += 1; + } + } + } let mut traces_kept = 0usize; let mut i = 0usize; while i < lines.len() && traces_kept < MAX_STACK_TRACES { - if stack_frame[i] { + if is_stack_frame(lines[i]) { let start = i; let mut taken = 0usize; - while i < lines.len() && stack_frame[i] { + while i < lines.len() && is_stack_frame(lines[i]) { if taken < STACK_TRACE_MAX_LINES { keep.insert(i); taken += 1; @@ -268,42 +240,34 @@ pub fn compress_signal(content: &str, block_tokens: bool) -> Option = kept_vec.into_iter().collect(); + let kept_set: std::collections::BTreeSet = kept_vec.into_iter().collect(); let mut out = String::with_capacity(content.len() / 2 + 64); - let mut any_token = false; let mut prev: Option = None; for &i in &kept_set { - let gap = match prev { - Some(p) => Some((p + 1, i)), - None if i > 0 => Some((0, i)), - None => None, - }; - if let Some((s, e)) = gap { - any_token |= emit_gap(&mut out, &lines, s, e, block_tokens); - } - out.push_str(lines[i]); - if let Some(a) = annotations.get(&i) { - out.push_str(a); + if let Some(p) = prev { + let gap = i - p - 1; + if gap > 0 { + let _ = writeln!(out, "[... {gap} line(s) omitted ...]"); + } + } else if i > 0 { + let _ = writeln!(out, "[... {i} line(s) omitted ...]"); } - out.push('\n'); + let _ = writeln!(out, "{}", lines[i]); prev = Some(i); } - if let Some(p) = prev - && p + 1 < lines.len() - { - any_token |= emit_gap(&mut out, &lines, p + 1, lines.len(), block_tokens); + if let Some(p) = prev { + let tail = lines.len().saturating_sub(p + 1); + if tail > 0 { + let _ = writeln!(out, "[... {tail} line(s) omitted ...]"); + } } - if any_token { - out.push_str(BLOCK_NOTE); - out.push('\n'); - } if out.len() >= content.len() { return None; } log::debug!( - "[tinyjuice][log] signal kept {} of {} line(s)", + "[tokenjuice][log] signal kept {} of {} line(s)", kept_set.len(), lines.len(), ); @@ -313,90 +277,153 @@ pub fn compress_signal(content: &str, block_tokens: bool) -> Option bool, - max_templates: usize, - keep_context: bool, - keep: &mut BTreeSet, - annotations: &mut HashMap, -) { - let mut order: Vec = Vec::new(); - let mut groups: HashMap> = HashMap::new(); - for (i, line) in lines.iter().enumerate() { - if is_member(i) { - let key = log_template::template_key(line); - if !groups.contains_key(&key) { - order.push(key.clone()); - } - groups.entry(key).or_default().push(i); - } +/// Lossless reformat for repetitive non-command logs. Consecutive runs of the +/// same line template are represented as one template plus captured variants. +pub fn compress_templates(content: &str) -> Option { + let lines: Vec<&str> = content.lines().collect(); + if lines.len() < TEMPLATE_MIN_RUN { + return None; } - for key in order.into_iter().take(max_templates) { - let idx = &groups[&key]; - let first = idx[0]; - let last = *idx.last().expect("non-empty group"); - keep.insert(first); - if idx.len() > 1 { - let ts = ( - log_template::parse_timestamp(lines[first]), - log_template::parse_timestamp(lines[last]), - ); - let ann = match ts { - (Some(a), Some(b)) => format!(" [×{} first {a}, last {b}]", idx.len()), - _ => format!(" [×{}]", idx.len()), + let mut out = String::with_capacity(content.len() / 2 + 128); + let mut i = 0usize; + let mut emitted_block = false; + + while i < lines.len() { + let Some(first) = line_template(lines[i]) else { + let _ = writeln!(out, "{}", lines[i]); + i += 1; + continue; + }; + + let start = i; + let mut captures = vec![first.captures]; + i += 1; + while i < lines.len() { + let Some(next) = line_template(lines[i]) else { + break; }; - annotations.insert(first, ann); - } - if keep_context { - for j in first.saturating_sub(CONTEXT_BEFORE)..first { - keep.insert(j); + if next.template != first.template || next.captures.len() != captures[0].len() { + break; } - let ctx_end = (first + CONTEXT_AFTER).min(lines.len().saturating_sub(1)); - for j in (first + 1)..=ctx_end { - keep.insert(j); + captures.push(next.captures); + i += 1; + } + + if captures.len() >= TEMPLATE_MIN_RUN { + write_template_block(&mut out, &first.template, &captures); + emitted_block = true; + } else { + for line in &lines[start..i] { + let _ = writeln!(out, "{line}"); } } } + + let out = out.trim_end().to_string(); + if !emitted_block || out.len() >= content.len() { + return None; + } + log::debug!( + "[tokenjuice][log] template reformat {} -> {} bytes", + content.len(), + out.len(), + ); + Some(CompressOutput::reformatted(out, CompressorKind::Log)) } -/// Emit a dropped `[start, end)` gap. Gaps of at most [`MIN_GAP`] lines are -/// written verbatim (a marker + token would be larger than the omitted text). -/// Larger gaps get the standard omission marker — carrying a CCR token that -/// makes the region recoverable — optionally preceded by template summaries -/// when the region is dominated by a few repeating templates. Returns whether a -/// retrieval token was emitted. -fn emit_gap( - out: &mut String, - lines: &[&str], - start: usize, - end: usize, - block_tokens: bool, -) -> bool { - let len = end - start; - if len <= MIN_GAP { - for line in &lines[start..end] { - let _ = writeln!(out, "{line}"); +#[derive(Debug, Clone, PartialEq, Eq)] +struct LineTemplate { + template: String, + captures: Vec, +} + +fn line_template(line: &str) -> Option { + let mut template = String::with_capacity(line.len()); + let mut captures = Vec::new(); + let mut chars = line.char_indices().peekable(); + + while let Some((idx, ch)) = chars.next() { + if ch == '0' && chars.peek().is_some_and(|(_, next)| *next == 'x') { + let start = idx; + let mut end = idx + ch.len_utf8(); + if let Some((x_idx, x_ch)) = chars.next() { + end = x_idx + x_ch.len_utf8(); + } + while let Some((next_idx, next_ch)) = chars.peek().copied() { + if next_ch.is_ascii_hexdigit() { + chars.next(); + end = next_idx + next_ch.len_utf8(); + } else { + break; + } + } + template.push_str("{}"); + captures.push(line[start..end].to_string()); + } else if ch.is_ascii_digit() { + let start = idx; + let mut end = idx + ch.len_utf8(); + while let Some((next_idx, next_ch)) = chars.peek().copied() { + if next_ch.is_ascii_digit() { + chars.next(); + end = next_idx + next_ch.len_utf8(); + } else { + break; + } + } + template.push_str("{}"); + captures.push(line[start..end].to_string()); + } else { + template.push(ch); } - return false; } - let region = &lines[start..end]; - if len >= MIN_REGION_FOR_TEMPLATES - && let Some(summary) = log_template::summarize_region(region) + + if captures.len() < 2 + || template.trim().len() < 12 + || template_constant_tokens(&template) < TEMPLATE_MIN_CONSTANT_TOKENS { - for s in summary { - let _ = writeln!(out, "{s}"); + return None; + } + Some(LineTemplate { template, captures }) +} + +fn template_constant_tokens(template: &str) -> usize { + template + .split(|ch: char| !ch.is_ascii_alphanumeric()) + .filter(|token| !token.is_empty()) + .count() +} + +fn write_template_block(out: &mut String, template: &str, captures: &[Vec]) { + let fields = captures.first().map_or(0, Vec::len); + let _ = writeln!( + out, + "[TOKENJUICE LOG TEMPLATE run={} fields={fields}]", + captures.len() + ); + let _ = writeln!(out, "{template}"); + for row in captures { + let rendered = row + .iter() + .map(|capture| escape_capture(capture)) + .collect::>() + .join("\t"); + let _ = writeln!(out, "{rendered}"); + } + let _ = writeln!(out, "[/TOKENJUICE LOG TEMPLATE]"); +} + +fn escape_capture(value: &str) -> String { + let mut out = String::with_capacity(value.len()); + for ch in value.chars() { + match ch { + '\\' => out.push_str("\\\\"), + '\t' => out.push_str("\\t"), + '\r' => out.push_str("\\r"), + _ => out.push(ch), } } - let token = block_token(®ion.join("\n"), block_tokens); - let _ = writeln!(out, "[... {len} line(s) omitted ...{token}]"); - !token.is_empty() + out } fn select_first_last(idx: &[usize], cap: usize) -> Vec { @@ -441,24 +468,27 @@ fn is_stack_frame(line: &str) -> bool { return false; } let indented = line.starts_with(' ') || line.starts_with('\t'); - // Rust backtrace frames: " 12: core::panicking::panic" - let rust_frame = || { - let digits = trimmed.chars().take_while(char::is_ascii_digit).count(); - digits > 0 && trimmed[digits..].starts_with(": ") - }; - // Go goroutine frames: "\t/path/file.go:42 +0x1b" - let go_frame = || trimmed.contains(".go:"); indented && (trimmed.starts_with("at ") || trimmed.starts_with("File \"") - || (trimmed.starts_with('#') && trimmed[1..].starts_with(|c: char| c.is_ascii_digit())) - || rust_frame() - || go_frame()) + || (trimmed.starts_with('#') && trimmed[1..].starts_with(|c: char| c.is_ascii_digit()))) +} + +fn normalize_for_dedupe(line: &str) -> String { + line.chars() + .filter(|c| !c.is_ascii_digit()) + .collect::() + .to_ascii_lowercase() + .split_whitespace() + .collect::>() + .join(" ") } #[cfg(test)] mod tests { use super::*; + use crate::cache::{CcrStore, MemoryCcrStore}; + use crate::types::{CompressOptions, ContentHint, ContentKind}; fn noisy_log() -> String { let mut s = String::new(); @@ -472,333 +502,232 @@ mod tests { s } - /// The lossless run-length fallback collapses byte-identical consecutive - /// lines into `line [×N]`, dropping nothing and preserving order. - #[test] - fn lossless_run_length_collapses_exact_duplicates() { - let mut s = String::from("start of stream\n"); - for _ in 0..50 { - s.push_str("2026-07-05T09:00:00Z INFO heartbeat ok\n"); + fn repetitive_template_log() -> String { + let mut s = String::new(); + for i in 0..80 { + let _ = writeln!( + s, + "2026-07-05T12:{:02}:00Z worker-{i} processed item id={} shard={}", + i % 60, + 10_000 + i, + i % 8 + ); } - s.push_str("2026-07-05T09:00:01Z INFO heartbeat ok differs\n"); - s.push_str("end of stream\n"); - let result = lossless_run_length(&s).expect("collapses"); - assert!(!result.lossy, "run-length collapse drops nothing"); - assert_eq!(result.kind, CompressorKind::Log); - let out = result.text; - // Non-duplicated lines survive verbatim. - assert!(out.contains("start of stream"), "{out}"); - assert!(out.contains("end of stream"), "{out}"); - assert!(out.contains("heartbeat ok differs"), "{out}"); - // The 50 identical lines collapse to one with an exact count. - assert!( - out.contains("2026-07-05T09:00:00Z INFO heartbeat ok [×50]"), - "run collapsed with count: {out}" - ); - assert_eq!( - out.matches("INFO heartbeat ok [×50]").count(), - 1, - "only one collapsed line: {out}" - ); - assert!(out.len() < s.len()); + s } - /// A run shorter than the break-even point is left verbatim (the marker - /// would not save bytes), and content with no runs declines. - #[test] - fn lossless_run_length_declines_without_savings() { - // Distinct lines: nothing to collapse. - let distinct = (0..80) - .map(|i| format!("line number {i} is unique")) - .collect::>() - .join("\n"); - assert!(lossless_run_length(&distinct).is_none()); - // A 2-line run of a very short line does not save with the marker. - let short = "ok\nok\ntail line here to keep it distinct\n"; - assert!(lossless_run_length(short).is_none()); + fn reconstruct_template_reformat(output: &str) -> String { + let mut out = String::new(); + let lines: Vec<&str> = output.lines().collect(); + let mut i = 0usize; + while i < lines.len() { + if lines[i].starts_with("[TOKENJUICE LOG TEMPLATE ") { + let template = lines[i + 1]; + i += 2; + while i < lines.len() && lines[i] != "[/TOKENJUICE LOG TEMPLATE]" { + let captures = lines[i] + .split('\t') + .map(unescape_capture_for_test) + .collect::>(); + let _ = writeln!(out, "{}", apply_template_for_test(template, &captures)); + i += 1; + } + i += 1; + } else { + let _ = writeln!(out, "{}", lines[i]); + i += 1; + } + } + out + } + + fn unescape_capture_for_test(value: &str) -> String { + let mut out = String::new(); + let mut chars = value.chars(); + while let Some(ch) = chars.next() { + if ch == '\\' { + match chars.next() { + Some('\\') => out.push('\\'), + Some('t') => out.push('\t'), + Some('r') => out.push('\r'), + Some(other) => { + out.push('\\'); + out.push(other); + } + None => out.push('\\'), + } + } else { + out.push(ch); + } + } + out } - #[test] - fn lossless_run_length_preserves_trailing_newline() { - let with_nl = "header line one\nx\nx\nx\nx\nx\ntail\n"; - let out = lossless_run_length(with_nl).expect("collapses").text; - assert!( - out.ends_with("tail\n"), - "trailing newline preserved: {out:?}" - ); - assert!(out.contains("x [×5]"), "{out}"); + fn apply_template_for_test(template: &str, captures: &[String]) -> String { + let mut out = String::new(); + let mut rest = template; + for capture in captures { + let Some((head, tail)) = rest.split_once("{}") else { + break; + }; + out.push_str(head); + out.push_str(capture); + rest = tail; + } + out.push_str(rest); + out } #[test] - fn signal_keeps_errors_and_summary_drops_noise() { - let input = noisy_log(); - let out = compress_signal(&input, false).expect("compresses").text; - assert!(out.contains("error[E0382]"), "{out}"); - assert!(out.contains("error: aborting"), "{out}"); - assert!(out.contains("test result: FAILED"), "{out}"); - assert!(out.len() < input.len()); - assert!(out.contains("omitted")); + fn template_reformat_is_lossless_and_smaller() { + let input = repetitive_template_log(); + let out = compress_templates(&input).expect("template reformat"); + + assert!(!out.lossy); + assert!(out.text.contains("[TOKENJUICE LOG TEMPLATE run=80")); + assert!(out.text.len() < input.len(), "{}", out.text); + assert_eq!( + reconstruct_template_reformat(&out.text).trim_end(), + input.trim_end() + ); } - #[test] - fn recognizes_rust_and_go_stack_frames() { - // Rust backtrace - assert!(is_stack_frame(" 12: core::panicking::panic")); - assert!(is_stack_frame(" 3: tinyjuice::compress::route")); - // Go goroutine trace - assert!(is_stack_frame("\t/srv/app/main.go:42 +0x1b")); - // JS / Python / gdb still recognized - assert!(is_stack_frame( - " at Object. (/app/index.js:1:1)" - )); - assert!(is_stack_frame( - " File \"/app/main.py\", line 3, in " - )); - assert!(is_stack_frame(" #4 0x0000 in main ()")); - // Non-frames - assert!(!is_stack_frame("12: not indented so not a frame")); - assert!(!is_stack_frame(" ordinary indented prose")); + #[tokio::test] + async fn template_reformat_routes_without_ccr() { + let input = repetitive_template_log(); + let hint = ContentHint { + explicit: Some(ContentKind::Log), + ..Default::default() + }; + let opts = CompressOptions { + min_bytes_to_compress: 64, + ccr_min_tokens: usize::MAX, + ..Default::default() + }; + + let out = crate::compress_content(&input, Some(hint), &opts).await; + + assert!(out.applied); + assert!(!out.lossy); + assert!(out.ccr_token.is_none()); + assert!(out.text.contains("[TOKENJUICE LOG TEMPLATE")); } #[test] - fn rust_backtrace_frames_survive_signal_compression() { - let mut s = String::new(); - for i in 0..200 { - let _ = writeln!(s, "request handled id={i} status=200"); - } - let _ = writeln!(s, "thread 'main' panicked at 'boom', src/main.rs:10:5"); - let _ = writeln!(s, "stack backtrace:"); - for i in 0..6 { - let _ = writeln!(s, " {i}: some_crate::module::function_{i}"); - } - let out = compress_signal(&s, false).expect("compresses").text; - assert!(out.contains("panicked"), "{out}"); + fn template_reformat_transform_runs_without_ccr() { + let input = repetitive_template_log(); + let pipeline_input = PipelineInput { + content: &input, + original_content: &input, + content_kind: ContentKind::Log, + original_bytes: input.len(), + }; + let transform = LogTemplateTransform; + + assert!(transform.applies_to(&pipeline_input)); + let out = transform.apply(&pipeline_input).expect("template reformat"); + + assert_eq!(out.kind, CompressorKind::Log); assert!( - out.contains("some_crate::module::function_2"), - "backtrace frames dropped:\n{out}" + out.text.contains("[TOKENJUICE LOG TEMPLATE"), + "{}", + out.text + ); + assert_eq!( + reconstruct_template_reformat(&out.text).trim_end(), + input.trim_end() ); } #[test] - fn non_log_data_passes_through() { - let mut s = String::new(); - for i in 0..400 { - let _ = writeln!(s, "/var/data/file_{i:04}.bin\t{i}\trwxr-xr-x"); - } - assert!(compress_signal(&s, false).is_none()); + fn template_reformat_transform_skips_non_log_input() { + let input = PipelineInput { + content: "plain text", + original_content: "plain text", + content_kind: ContentKind::PlainText, + original_bytes: "plain text".len(), + }; + let transform = LogTemplateTransform; + + assert!(!transform.applies_to(&input)); } #[test] - fn gap_token_retrieves_omitted_lines() { - use crate::cache; + fn signal_log_transform_requires_retained_ccr() { let input = noisy_log(); - let out = compress_signal(&input, true).expect("compresses").text; - let tokens = cache::parse_markers(&out); - assert!(!tokens.is_empty(), "gap markers should carry tokens: {out}"); - let block = cache::retrieve(&tokens[0]).expect("stored"); - assert!( - block.contains("Compiling crate_"), - "gap block should hold the omitted noise: {block}" - ); - } + let pipeline_input = PipelineInput { + content: &input, + original_content: &input, + content_kind: ContentKind::Log, + original_bytes: input.len(), + }; + let transform = SignalLogTransform; - #[test] - fn distinct_error_templates_all_kept_with_counts() { - // 150 lines of noise, then three distinct error templates, each repeated. - let mut s = String::new(); - for i in 0..150 { - let _ = writeln!(s, "081109 20{i:04} 12 INFO worker: handled request {i} ok"); - } - for i in 0..5 { - let _ = writeln!( - s, - "081110 00{i:04} 12 ERROR disk: write to /dev/sda{i} failed" - ); - } - for i in 0..5 { - let _ = writeln!( - s, - "081110 01{i:04} 12 ERROR net: connection to host_{i} refused" - ); - } - for i in 0..5 { - let _ = writeln!( - s, - "081110 02{i:04} 12 ERROR auth: token for user_{i} expired" - ); - } - let out = compress_signal(&s, false).expect("compresses").text; - assert!(out.contains("write to /dev/sda"), "disk error kept: {out}"); - assert!(out.contains("connection to host_"), "net error kept: {out}"); - assert!(out.contains("token for user_"), "auth error kept: {out}"); - // Each distinct template collapses to one exemplar with an ×5 count. - assert_eq!(out.matches("[×5").count(), 3, "one ×5 per template: {out}"); - } + let rejecting_store = MemoryCcrStore::new(1, 1); + assert!(transform.apply(&pipeline_input, &rejecting_store).is_none()); - #[test] - fn burst_of_templated_errors_collapses_to_one_exemplar() { - let mut s = String::new(); - for i in 0..120 { - let _ = writeln!(s, "12:00:{:02} INFO idle tick {i}", i % 60); - } - for i in 0..20 { - let _ = writeln!( - s, - "13:37:{:02} ERROR handler: request blk_{i} timed out after 30s", - i % 60 - ); - } - let out = compress_signal(&s, false).expect("compresses").text; - // The 20-line burst collapses to a single exemplar annotated ×20 with - // first/last timestamps. A couple of context lines around the exemplar - // and a gap template summary may echo the phrase, but the 20 verbatim - // copies are gone. - assert!(out.contains("[×20 first 13:37:00, last 13:37:19]"), "{out}"); - assert!( - out.matches("13:37:").count() < 20, - "burst should not keep 20 verbatim copies: {out}" - ); + let store = MemoryCcrStore::default(); + let out = transform + .apply(&pipeline_input, &store) + .expect("retained signal log"); + + assert_eq!(out.kind(), CompressorKind::Log); + assert!(out.text().contains("error[E0382]"), "{}", out.text()); + assert_eq!(store.get(out.token()).as_deref(), Some(input.as_str())); } #[test] - fn context_lines_kept_around_error() { - let mut s = String::new(); - for i in 0..150 { - let _ = writeln!(s, "12:00:{:02} INFO steady state {i}", i % 60); - } - let _ = writeln!(s, "13:00:00 INFO before-2 opening connection"); - let _ = writeln!(s, "13:00:01 INFO before-1 sending handshake"); - let _ = writeln!(s, "13:00:02 ERROR handshake rejected by peer"); - let _ = writeln!(s, "13:00:03 INFO after-1 closing socket"); - let _ = writeln!(s, "13:00:04 INFO after-2 cleaning up"); - for i in 0..50 { - let _ = writeln!(s, "13:10:{:02} INFO steady state {i}", i % 60); - } - let out = compress_signal(&s, false).expect("compresses").text; - assert!(out.contains("handshake rejected"), "error kept: {out}"); - assert!( - out.contains("before-1 sending handshake"), - "ctx-1 before: {out}" - ); - assert!( - out.contains("before-2 opening connection"), - "ctx-2 before: {out}" - ); - assert!(out.contains("after-1 closing socket"), "ctx-1 after: {out}"); - assert!(out.contains("after-2 cleaning up"), "ctx-2 after: {out}"); + fn signal_log_transform_skips_non_log_input() { + let input = PipelineInput { + content: "plain text", + original_content: "plain text", + content_kind: ContentKind::PlainText, + original_bytes: "plain text".len(), + }; + let transform = SignalLogTransform; + let store = MemoryCcrStore::default(); + + assert_eq!(transform.estimate_bloat(&input), 0.0); + assert!(transform.apply(&input, &store).is_none()); } #[test] - fn short_gap_emitted_verbatim() { - // A 1-line gap between two kept errors must stay verbatim, not become a - // marker + token that is larger than the omitted line. - let mut s = String::new(); - for i in 0..150 { - let _ = writeln!(s, "081109 20{i:04} 12 INFO noise line {i}"); - } - let _ = writeln!(s, "081110 000001 12 ERROR alpha subsystem failed hard"); - let _ = writeln!(s, "081110 000002 12 INFO a single quiet interstitial line"); - let _ = writeln!(s, "081110 000003 12 ERROR beta subsystem failed hard"); - for i in 0..30 { - let _ = writeln!(s, "081110 01{i:04} 12 INFO tail noise {i}"); + fn template_reformat_declines_without_dynamic_fields() { + let mut input = String::new(); + for _ in 0..40 { + let _ = writeln!(input, "plain repeated line without dynamic fields"); } - let out = compress_signal(&s, true).expect("compresses").text; - // The single interstitial line survives verbatim between the two errors, - // with no omission marker separating them. - assert!( - out.contains("a single quiet interstitial line"), - "1-line gap should be verbatim: {out}" - ); - let alpha = out.find("alpha subsystem").expect("alpha kept"); - let beta = out.find("beta subsystem").expect("beta kept"); - let between = &out[alpha..beta]; - assert!( - !between.contains("omitted"), - "no marker for a 1-line gap: {between}" - ); + + assert!(compress_templates(&input).is_none()); } #[test] - fn warn_exception_deduped_not_kept_verbatim() { - // The HDFS pathology: many `WARN … Got exception …` lines. They must be - // treated as warnings and collapsed to a single deduped exemplar, not - // kept verbatim as distinct errors. - let mut s = String::new(); - for i in 0..150 { - let _ = writeln!(s, "081109 20{i:04} 12 INFO worker: processed block {i}"); + fn template_reformat_declines_low_context_dynamic_lines() { + let mut input = String::new(); + for i in 0..40 { + let _ = writeln!(input, "id={} value={}", 10_000 + i, 20_000 + i); } - for i in 0..30 { - let _ = writeln!( - s, - "081109 21{i:04} {i} WARN dfs.DataNode$DataXceiver: 10.0.0.{i}:50010:Got exception while serving blk_-{i}00 to /10.1.0.{i}:" - ); - } - let out = compress_signal(&s, false).expect("compresses").text; - assert_eq!( - out.matches("Got exception while serving").count(), - 1, - "WARN+exception lines must dedupe to one exemplar: {out}" - ); + + assert!(compress_templates(&input).is_none()); } #[test] - fn rare_lowercase_fail_line_survives_pipe_delimited_noise() { - // HealthApp/logcat shape: `ts|component|pid|message`, no level tokens, - // and a rare failure line using bare lowercase `fail`. It must survive - // as a warning exemplar instead of being swallowed into an omitted gap. - let mut s = String::new(); - for i in 0..300 { - let _ = writeln!( - s, - "20171223-22:19:{:02}:{:03}|HiH_HiHealthDataInsertStore|30002312|saveStatData() type ={},time = 1513958400000,statClient = 2,who is 1", - i % 60, - i % 1000, - 40000 + i - ); - } - let _ = writeln!( - s, - "20171223-22:19:58:380|HiH_HiHealthDataInsertStore|30002312|saveHealthDetailData() saveOneDetailData fail hiHealthData = 1513958400000,type = 40003" - ); - for i in 0..100 { - let _ = writeln!( - s, - "20171223-22:20:{:02}:{:03}|HiH_DataStatManager|30002312|new date =20171223, type={},7163.0,old=6983.0", - i % 60, - i % 1000, - 40000 + i - ); - } - let _ = writeln!( - s, - "20171223-22:21:00:381|HiH_HiHealthDataInsertStore|30002312|saveHealthDetailData() saveOneDetailData fail hiHealthData = 1513958400000,type = 40005" - ); - let out = compress_signal(&s, false).expect("compresses").text; - assert!( - out.contains("saveOneDetailData fail"), - "rare lowercase fail line must survive: {out}" - ); - assert!(out.len() < s.len(), "must shrink"); + fn signal_keeps_errors_and_summary_drops_noise() { + let input = noisy_log(); + let out = compress_signal(&input).expect("compresses").text; + assert!(out.contains("error[E0382]"), "{out}"); + assert!(out.contains("error: aborting"), "{out}"); + assert!(out.contains("test result: FAILED"), "{out}"); + assert!(out.len() < input.len()); + assert!(out.contains("omitted")); } #[test] - fn round_trips_losslessly_with_ccr() { - // Every omitted region is recoverable through its marker token. + fn non_log_data_passes_through() { let mut s = String::new(); - for i in 0..300 { - let _ = writeln!( - s, - "081109 20{i:04} 12 INFO dfs.DataNode: block blk_{i} stored ok" - ); - } - let _ = writeln!(s, "081110 000000 12 ERROR fatal corruption detected"); - let out = compress_signal(&s, true).expect("compresses"); - assert!(out.lossy, "signal compression is lossy"); - assert!(out.text.len() < s.len(), "must shrink"); - let tokens = crate::cache::parse_markers(&out.text); - assert!(!tokens.is_empty(), "recovery tokens present: {}", out.text); - for t in &tokens { - assert!(crate::cache::retrieve(t).is_some(), "token {t} recoverable"); + for i in 0..400 { + let _ = writeln!(s, "/var/data/file_{i:04}.bin\t{i}\trwxr-xr-x"); } + assert!(compress_signal(&s).is_none()); } } diff --git a/src/compressors/ml_text.rs b/src/compressors/ml_text.rs index 0b2e027..12759bd 100644 --- a/src/compressors/ml_text.rs +++ b/src/compressors/ml_text.rs @@ -11,10 +11,10 @@ //! runtime is unavailable, `compress` declines so the router falls back to the //! generic compressor — never an error in the agent loop. -use async_trait::async_trait; - -use super::{Compressor, tag_protect}; +use super::Compressor; +use crate::compressors::text::compress_ml_with_tag_protection; use crate::types::{CompressInput, CompressOptions, CompressOutput, CompressorKind}; +use async_trait::async_trait; pub struct MlTextCompressor; @@ -29,35 +29,7 @@ impl Compressor for MlTextCompressor { input: &CompressInput<'_>, opts: &CompressOptions, ) -> Option { - if !opts.ml_text_enabled { - return None; - } - // Structural XML-ish tags (e.g. ``, ``) - // are markers downstream code parses; swap them for opaque - // placeholders so the lossy model can't destroy them, and splice them - // back afterwards. - let (protected, saved) = tag_protect::protect(input.content); - match crate::ml::compress(&protected, opts).await { - Ok(Some(text)) => { - if !tag_protect::all_placeholders_present(&text, &saved) { - log::debug!( - "[tinyjuice][ml] output lost a protected structural tag; declining" - ); - return None; - } - let restored = tag_protect::restore(&text, &saved); - if restored.len() < input.content.len() { - Some(CompressOutput::lossy(restored, CompressorKind::MlText)) - } else { - None - } - } - Ok(None) => None, - Err(e) => { - log::debug!("[tinyjuice][ml] unavailable, falling back: {e:#}"); - None - } - } + compress_ml_with_tag_protection(input.content, opts).await } } diff --git a/src/compressors/mod.rs b/src/compressors/mod.rs index 01ba58d..cd08438 100644 --- a/src/compressors/mod.rs +++ b/src/compressors/mod.rs @@ -5,13 +5,8 @@ //! changed hunks in diffs, signatures in code, anomalous rows in JSON — and //! drops the rest. Whole-payload recovery belongs to the router //! ([`crate::compress`]), which offloads the original to the CCR cache and -//! appends a retrieval footer. In addition, compressors may offload each -//! *omitted block* individually (via [`block_token`]) so its omission marker -//! carries a token that retrieves exactly that block — gated on -//! `opts.ccr_enabled`. +//! appends a retrieval footer. -pub(crate) mod adaptive; -pub(crate) mod anchors; pub mod code; pub mod diff; pub mod generic; @@ -22,7 +17,8 @@ pub mod log_template; pub mod ml_text; pub mod search; pub mod signals; -pub(crate) mod tag_protect; +pub mod text; +pub mod web_extract; use async_trait::async_trait; @@ -52,16 +48,13 @@ static LOG_COMPRESSOR: log::LogCompressor = log::LogCompressor; static SEARCH_COMPRESSOR: search::SearchCompressor = search::SearchCompressor; static DIFF_COMPRESSOR: diff::DiffCompressor = diff::DiffCompressor; static HTML_COMPRESSOR: html::HtmlCompressor = html::HtmlCompressor; -static ML_TEXT_COMPRESSOR: ml_text::MlTextCompressor = ml_text::MlTextCompressor; +static TEXT_COMPRESSOR: text::TextCompressor = text::TextCompressor; static GENERIC_COMPRESSOR: generic::GenericCompressor = generic::GenericCompressor; /// Map a detected [`ContentKind`] to the compressor that handles it. /// -/// `PlainText` routes to the ML compressor; whether it actually runs is gated -/// by `opts.ml_text_enabled` (and runtime Python/runtime_python_server -/// availability), and it falls back to [`generic::GenericCompressor`] otherwise -/// — that gating lives in [`crate::compress`], so this -/// function is a pure static mapping. +/// `PlainText` routes to the text compressor, which tries the optional ML path +/// first and then the deterministic extractive TextCrusher fallback. pub fn compressor_for(kind: ContentKind) -> &'static dyn Compressor { match kind { ContentKind::Json => &JSON_COMPRESSOR, @@ -70,7 +63,7 @@ pub fn compressor_for(kind: ContentKind) -> &'static dyn Compressor { ContentKind::Search => &SEARCH_COMPRESSOR, ContentKind::Diff => &DIFF_COMPRESSOR, ContentKind::Html => &HTML_COMPRESSOR, - ContentKind::PlainText => &ML_TEXT_COMPRESSOR, + ContentKind::PlainText => &TEXT_COMPRESSOR, } } @@ -80,26 +73,6 @@ pub fn generic_compressor() -> &'static dyn Compressor { &GENERIC_COMPRESSOR } -/// One-line note appended to compacted output when any omitted block carries -/// its own retrieval token. -pub(crate) const BLOCK_NOTE: &str = "\n[omitted blocks are individually retrievable: call tinyjuice_retrieve with the token inside an omission marker to expand just that block]"; - -/// Offload an omitted block to CCR and return the marker text to embed in the -/// omission placeholder (` ⟦tj:⟧`), or an empty string when per-block -/// tokens are disabled or the block couldn't be retained. The store is -/// content-addressed and idempotent, so identical blocks share one entry. -pub(crate) fn block_token(block: &str, enabled: bool) -> String { - if !enabled { - return String::new(); - } - let (token, retained) = crate::cache::offload_checked(block); - if retained { - format!(" {}", crate::cache::format_marker(&token)) - } else { - String::new() - } -} - #[cfg(test)] mod tests { use super::*; @@ -113,7 +86,7 @@ mod tests { (ContentKind::Search, CompressorKind::Search), (ContentKind::Diff, CompressorKind::Diff), (ContentKind::Html, CompressorKind::Html), - (ContentKind::PlainText, CompressorKind::MlText), + (ContentKind::PlainText, CompressorKind::TextCrusher), ]; for (kind, expected) in cases { diff --git a/src/compressors/search.rs b/src/compressors/search.rs index 94b4603..da951c2 100644 --- a/src/compressors/search.rs +++ b/src/compressors/search.rs @@ -6,7 +6,8 @@ //! full result set to CCR so the complete list is one `retrieve` away. //! //! Ranking: when the caller supplies a `query` in the [`ContentHint`], matches -//! are scored by query-term density in the body; otherwise by body length / +//! are scored by the shared BM25/tokenizer path so regex punctuation and exact +//! identifiers carry through. Without a query, matches use body length / //! uniqueness (longer, more-distinctive lines first), with importance signals //! (error/TODO) always boosted. Group/file order is preserved (first-seen). //! @@ -15,26 +16,82 @@ //! lossless by the CCR offload — and is enabled by default per project decision. use async_trait::async_trait; -use std::collections::HashMap; +use std::borrow::Cow; use std::fmt::Write as _; -use super::adaptive::compute_optimal_k; +use super::Compressor; use super::signals::line_score; -use super::{BLOCK_NOTE, Compressor, block_token}; +use crate::cache::CcrStore; use crate::detect::parse_search_line; -use crate::types::{CompressInput, CompressOptions, CompressOutput, CompressorKind}; +use crate::pipeline::{OffloadOutput, OffloadTransform, PipelineInput, estimate_bloat}; +use crate::relevance::{Bm25Corpus, tokenize}; +use crate::types::LineRange; +use crate::types::{CompressInput, CompressOptions, CompressOutput, CompressorKind, ContentKind}; /// Only compress result sets with more than this many matching lines. pub const MIN_MATCHES: usize = 40; -/// Floor on matches kept per file before the "+N more" tally. The actual -/// keep count adapts to the diversity of each file's matches (see -/// [`compute_optimal_k`]) between this and [`MAX_K_PER_FILE`]. +/// Matches kept per file before the "+N more" tally. pub const TOP_K_PER_FILE: usize = 5; -/// Ceiling on matches kept per file when the matches are highly diverse. -pub const MAX_K_PER_FILE: usize = 12; +/// Default line context on each side for ranked search-read snippets. +pub const DEFAULT_SNIPPET_CONTEXT: usize = 2; pub struct SearchCompressor; +/// Typed offload transform for lossy search-result thinning. +#[derive(Debug, Clone, Default)] +pub struct SearchTransform { + query: Option, +} + +impl SearchTransform { + pub fn new() -> Self { + Self::default() + } + + pub fn with_query(mut self, query: impl Into) -> Self { + self.query = Some(query.into()); + self + } + + pub fn query(&self) -> Option<&str> { + self.query.as_deref() + } +} + +impl OffloadTransform for SearchTransform { + fn name(&self) -> &'static str { + "search" + } + + fn estimate_bloat(&self, input: &PipelineInput<'_>) -> f32 { + if input.content_kind != ContentKind::Search { + return 0.0; + } + let score = f32::from(estimate_bloat(input.content, input.content_kind).score) / 100.0; + if self + .query + .as_deref() + .is_some_and(|query| !query.trim().is_empty()) + { + score.max(0.1) + } else { + score + } + } + + fn apply(&self, input: &PipelineInput<'_>, store: &dyn CcrStore) -> Option { + if input.content_kind != ContentKind::Search { + return None; + } + let compacted = compress(input.content, self.query())?; + OffloadOutput::from_retained_put( + compacted.text, + CompressorKind::Search, + store.put(input.original_content), + ) + } +} + #[async_trait] impl Compressor for SearchCompressor { fn kind(&self) -> CompressorKind { @@ -44,9 +101,9 @@ impl Compressor for SearchCompressor { async fn compress( &self, input: &CompressInput<'_>, - opts: &CompressOptions, + _opts: &CompressOptions, ) -> Option { - compress(input.content, input.hint.query.as_deref(), opts.ccr_enabled) + compress(input.content, input.hint.query.as_deref()) } } @@ -57,94 +114,77 @@ struct Match<'a> { raw: &'a str, } -/// Compress search output. `query` (when known) ranks matches by term density. -/// With `block_tokens`, each per-file `+N more` tally carries a token that -/// retrieves exactly the omitted matches for that file. -pub fn compress(content: &str, query: Option<&str>, block_tokens: bool) -> Option { +struct ParsedMatch<'a> { + path: &'a str, + line_no: u64, + body: &'a str, + raw: &'a str, +} + +/// Compress search output. `query` (when known) ranks matches by shared BM25. +pub fn compress(content: &str, query: Option<&str>) -> Option { // Preserve any non-match preamble/summary lines (e.g. "80 match(es)") and // group match lines by file in first-seen order. let mut preamble: Vec<&str> = Vec::new(); - let mut files: Vec<(&str, Vec>)> = Vec::new(); - // Index alongside the Vec (which preserves first-seen file order): - // a linear scan per match is quadratic on repo-wide result sets. - let mut file_index: HashMap<&str, usize> = HashMap::new(); - let mut match_count = 0usize; - let mut unparsed_dropped = 0usize; - - let query_terms: Vec = query - .map(|q| { - q.split_whitespace() - .map(|t| t.to_ascii_lowercase()) - .filter(|t| t.len() >= 2) - .collect() - }) - .unwrap_or_default(); + let mut parsed_matches: Vec> = Vec::new(); for line in content.lines() { match parse_search_line(line) { Some((path, line_no, body)) => { - match_count += 1; - let score = score_match(body, &query_terms); - let m = Match { + parsed_matches.push(ParsedMatch { + path, line_no, body, - score, raw: line, - }; - if let Some(&idx) = file_index.get(path) { - files[idx].1.push(m); - } else { - file_index.insert(path, files.len()); - files.push((path, vec![m])); - } + }); } None => { - if !line.trim().is_empty() { - if files.is_empty() { - // Only keep preamble that appears before any match. - preamble.push(line); - } else { - // Context lines (grep -C), separators, trailing - // summaries — dropped, but tallied so the reader - // knows they existed. - unparsed_dropped += 1; - } + if !line.trim().is_empty() && parsed_matches.is_empty() { + // Only keep preamble that appears before any match. + preamble.push(line); } } } } - if match_count < MIN_MATCHES || files.is_empty() { + let match_count = parsed_matches.len(); + if match_count < MIN_MATCHES || parsed_matches.is_empty() { return None; } + let scores = score_matches(&parsed_matches, query); + let mut files: Vec<(&str, Vec>)> = Vec::new(); + for (parsed, score) in parsed_matches.iter().zip(scores) { + let m = Match { + line_no: parsed.line_no, + body: parsed.body, + score, + raw: parsed.raw, + }; + if let Some((_, v)) = files.iter_mut().find(|(p, _)| *p == parsed.path) { + v.push(m); + } else { + files.push((parsed.path, vec![m])); + } + } + let mut out = String::with_capacity(content.len() / 2 + 64); for line in &preamble { let _ = writeln!(out, "{line}"); } - let omitted_note = if unparsed_dropped > 0 { - format!(" · {unparsed_dropped} context/summary line(s) omitted") - } else { - String::new() - }; let _ = writeln!( out, - "[search: {} match(es) across {} file(s) · top {}-{} per file (adaptive){} · full set via retrieve footer]", + "[search: {} match(es) across {} file(s) · top {} per file · full set via retrieve footer]", match_count, files.len(), - TOP_K_PER_FILE, - MAX_K_PER_FILE, - omitted_note + TOP_K_PER_FILE ); - let mut any_token = false; + let mut omitted_total = 0usize; + let mut files_with_omissions = 0usize; for (path, mut matches) in files { let total = matches.len(); - // Adapt the keep count to this file's match diversity: redundant - // matches collapse toward the floor, diverse ones keep more. - let bodies: Vec<&str> = matches.iter().map(|m| m.body).collect(); - let top_k = compute_optimal_k(&bodies, TOP_K_PER_FILE, MAX_K_PER_FILE); - if total <= top_k { + if total <= TOP_K_PER_FILE { // Keep all in original (line-number) order. matches.sort_by_key(|m| m.line_no); for m in &matches { @@ -159,29 +199,32 @@ pub fn compress(content: &str, query: Option<&str>, block_tokens: bool) -> Optio .partial_cmp(&a.score) .unwrap_or(std::cmp::Ordering::Equal) }); - let mut kept: Vec<&Match<'_>> = matches.iter().take(top_k).collect(); + let mut kept: Vec<&Match<'_>> = matches.iter().take(TOP_K_PER_FILE).collect(); kept.sort_by_key(|m| m.line_no); for m in &kept { let _ = writeln!(out, "{}:{}:{}", path, m.line_no, m.body); } - // Offload this file's omitted matches (line order) behind one token. - let mut omitted: Vec<&Match<'_>> = matches.iter().skip(top_k).collect(); - omitted.sort_by_key(|m| m.line_no); - let omitted_raw = omitted.iter().map(|m| m.raw).collect::>().join("\n"); - let token = block_token(&omitted_raw, block_tokens); - any_token |= !token.is_empty(); - let _ = writeln!(out, "[+{} more match(es) in {path}{token}]", total - top_k); + let _ = writeln!( + out, + "[+{} more match(es) in {path}]", + total - TOP_K_PER_FILE + ); + omitted_total += total - TOP_K_PER_FILE; + files_with_omissions += 1; } - - let mut out = out.trim_end().to_string(); - if any_token { - out.push_str(BLOCK_NOTE); + if omitted_total > 0 { + let _ = writeln!( + out, + "[search omitted: {omitted_total} match(es) not shown across {files_with_omissions} file(s)]" + ); } + + let out = out.trim_end().to_string(); if out.len() >= content.len() { return None; } log::debug!( - "[tinyjuice][search] {} matches -> {} bytes (from {} bytes)", + "[tokenjuice][search] {} matches -> {} bytes (from {} bytes)", match_count, out.len(), content.len() @@ -189,24 +232,289 @@ pub fn compress(content: &str, query: Option<&str>, block_tokens: bool) -> Optio Some(CompressOutput::lossy(out, CompressorKind::Search)) } -/// Score a match body. With query terms, density of those terms dominates; -/// otherwise distinctiveness (length) with an importance bump for error/TODO. -fn score_match(body: &str, query_terms: &[String]) -> f32 { +/// Score search-result bodies. With query text, shared BM25 dominates so query +/// syntax and exact identifiers are interpreted the same way as search-read +/// candidate ranking. Otherwise, distinctiveness (length) plus importance +/// signals decides ranking. +fn score_matches(matches: &[ParsedMatch<'_>], query: Option<&str>) -> Vec { + if let Some(query) = query + && !tokenize(query).is_empty() + { + let corpus = Bm25Corpus::new(matches.iter().map(|m| m.body)); + return corpus + .score_all(query) + .into_iter() + .map(|score| { + let importance = line_score(matches[score.index].body); + score.score.max(importance * 0.25) + }) + .collect(); + } + + matches + .iter() + .map(|m| score_match_without_query(m.body)) + .collect() +} + +fn score_match_without_query(body: &str) -> f32 { let importance = line_score(body); - if query_terms.is_empty() { - // No query: favour longer, more-distinctive lines, plus importance. - let len_score = (body.trim().len() as f32 / 80.0).min(1.0); - return importance.max(0.2 + 0.8 * len_score); - } - let lower = body.to_ascii_lowercase(); - let hits = query_terms.iter().filter(|t| lower.contains(*t)).count(); - let density = hits as f32 / query_terms.len() as f32; - importance.max(density) + let len_score = (body.trim().len() as f32 / 80.0).min(1.0); + importance.max(0.2 + 0.8 * len_score) +} + +/// Host-provided search-read query metadata. TinyJuice only scores already +/// discovered matches; filesystem traversal stays in the host. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct SearchReadQuery { + pub literal: Option, + pub regex: Option, + pub symbols: Vec, + pub file_kinds: Vec, + pub penalize_vendor: bool, + pub penalize_generated: bool, +} + +impl Default for SearchReadQuery { + fn default() -> Self { + Self { + literal: None, + regex: None, + symbols: Vec::new(), + file_kinds: Vec::new(), + penalize_vendor: true, + penalize_generated: true, + } + } +} + +/// One candidate file already discovered by a host search adapter. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct SearchReadCandidate<'a> { + pub path: Cow<'a, str>, + pub matched_lines: Vec>, + pub imports: Vec>, + pub exports: Vec>, + pub generated: bool, + pub vendor: bool, + pub max_line: usize, +} + +impl<'a> SearchReadCandidate<'a> { + pub fn new(path: impl Into>) -> Self { + Self { + path: path.into(), + matched_lines: Vec::new(), + imports: Vec::new(), + exports: Vec::new(), + generated: false, + vendor: false, + max_line: 0, + } + } +} + +/// One host-provided match line. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct SearchReadLine<'a> { + pub line_number: usize, + pub text: Cow<'a, str>, +} + +impl<'a> SearchReadLine<'a> { + pub fn new(line_number: usize, text: impl Into>) -> Self { + Self { + line_number, + text: text.into(), + } + } +} + +/// Bounded snippet-window selection for a candidate file. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct SnippetWindowSelection { + pub windows: Vec, + pub omitted_matches: usize, +} + +/// Score a candidate file for a ranked search-read adapter. Exact symbol +/// matches dominate path matches, which dominate match density. +pub fn rank_search_read_candidate( + candidate: &SearchReadCandidate<'_>, + query: &SearchReadQuery, +) -> f32 { + let exact_symbol_match = query + .symbols + .iter() + .filter(|symbol| !symbol.trim().is_empty()) + .any(|symbol| candidate_has_exact_symbol(candidate, symbol)); + let path_match = query_terms(query) + .iter() + .any(|term| candidate.path.to_ascii_lowercase().contains(term)); + let regex_density = regex_match_density(candidate, query); + let import_export_match = query + .symbols + .iter() + .any(|symbol| import_export_contains(candidate, symbol)); + + let generated_penalty = if query.penalize_generated + && (candidate.generated || looks_generated_path(&candidate.path)) + { + 4.0 + } else { + 0.0 + }; + let vendor_penalty = + if query.penalize_vendor && (candidate.vendor || looks_vendor_path(&candidate.path)) { + 3.0 + } else { + 0.0 + }; + + (if exact_symbol_match { 8.0 } else { 0.0 }) + + (if path_match { 4.0 } else { 0.0 }) + + regex_density * 3.0 + + (if import_export_match { 2.0 } else { 0.0 }) + - generated_penalty + - vendor_penalty +} + +/// Select and merge snippet windows around match lines. The output is bounded +/// by `max_snippets`, with omitted match count made explicit. +pub fn select_snippet_windows( + match_lines: &[usize], + context: usize, + max_line: usize, + max_snippets: usize, +) -> SnippetWindowSelection { + if max_snippets == 0 || match_lines.is_empty() || max_line == 0 { + return SnippetWindowSelection { + windows: Vec::new(), + omitted_matches: match_lines.len(), + }; + } + + let mut lines: Vec = match_lines + .iter() + .copied() + .filter(|line| *line > 0) + .collect(); + lines.sort_unstable(); + lines.dedup(); + + let mut merged: Vec = Vec::new(); + for line in lines { + let range = LineRange::new( + line.saturating_sub(context).max(1), + (line + context).min(max_line), + ); + if let Some(last) = merged.last_mut() + && range.start <= last.end.saturating_add(1) + { + last.end = last.end.max(range.end); + continue; + } + merged.push(range); + } + + let omitted_matches = if merged.len() > max_snippets { + let kept_end = merged[max_snippets - 1].end; + match_lines.iter().filter(|line| **line > kept_end).count() + } else { + 0 + }; + merged.truncate(max_snippets); + + SnippetWindowSelection { + windows: merged, + omitted_matches, + } +} + +fn candidate_has_exact_symbol(candidate: &SearchReadCandidate<'_>, symbol: &str) -> bool { + candidate + .matched_lines + .iter() + .any(|line| contains_exact_token(&line.text, symbol)) + || candidate + .exports + .iter() + .any(|export| export.as_ref() == symbol) +} + +fn import_export_contains(candidate: &SearchReadCandidate<'_>, symbol: &str) -> bool { + candidate + .imports + .iter() + .chain(candidate.exports.iter()) + .any(|entry| entry.contains(symbol)) +} + +fn regex_match_density(candidate: &SearchReadCandidate<'_>, query: &SearchReadQuery) -> f32 { + if candidate.matched_lines.is_empty() { + return 0.0; + } + let terms = query_terms(query); + if terms.is_empty() && query.regex.is_none() { + return 0.0; + } + let hit_count = candidate + .matched_lines + .iter() + .filter(|line| { + let lower = line.text.to_ascii_lowercase(); + terms.iter().any(|term| lower.contains(term)) + || query + .regex + .as_ref() + .is_some_and(|regex| lower.contains(®ex.to_ascii_lowercase())) + }) + .count(); + hit_count as f32 / candidate.matched_lines.len() as f32 +} + +fn query_terms(query: &SearchReadQuery) -> Vec { + query + .literal + .iter() + .chain(query.symbols.iter()) + .flat_map(|value| tokenize(value)) + .collect() +} + +fn contains_exact_token(text: &str, needle: &str) -> bool { + tokenize(text) + .iter() + .any(|token| token == &needle.to_ascii_lowercase()) +} + +fn looks_vendor_path(path: &str) -> bool { + let lower = path.to_ascii_lowercase(); + lower.contains("/node_modules/") + || lower.contains("/vendor/") + || lower.contains("/third_party/") + || lower.starts_with("vendor/") + || lower.starts_with("third_party/") +} + +fn looks_generated_path(path: &str) -> bool { + let lower = path.to_ascii_lowercase(); + lower.contains("/target/") + || lower.contains("/dist/") + || lower.contains("/build/") + || lower.ends_with(".min.js") + || lower.ends_with(".min.css") + || lower.ends_with(".map") + || lower.ends_with("package-lock.json") + || lower.ends_with("cargo.lock") + || lower.ends_with("go.sum") } #[cfg(test)] mod tests { use super::*; + use crate::cache::{CcrStore, MemoryCcrStore}; + use crate::pipeline::PipelineInput; fn big_results() -> String { let mut s = String::from("120 match(es); scanned 3 file(s)\n"); @@ -222,9 +530,13 @@ mod tests { #[test] fn keeps_top_k_per_file_and_tally() { let input = big_results(); - let out = compress(&input, None, false).expect("compresses").text; + let out = compress(&input, None).expect("compresses").text; assert!(out.contains("more match(es) in src/a.rs"), "{out}"); assert!(out.contains("more match(es) in src/b.rs")); + assert!( + out.contains("[search omitted: 110 match(es) not shown across 2 file(s)]"), + "{out}" + ); // Preamble survives. assert!(out.contains("120 match(es)")); assert!(out.len() < input.len()); @@ -241,9 +553,7 @@ mod tests { }; let _ = writeln!(s, "src/x.rs:{i}:{body}"); } - let out = compress(&s, Some("needle token"), false) - .expect("compresses") - .text; + let out = compress(&s, Some("needle token")).expect("compresses").text; assert!( out.contains("special needle token"), "ranked-in match missing:\n{out}" @@ -251,132 +561,173 @@ mod tests { } #[test] - fn adaptive_k_keeps_floor_for_redundant_matches() { + fn query_ranking_tokenizes_regex_identifier_context() { let mut s = String::new(); for i in 0..50 { - let _ = writeln!(s, "src/r.rs:{i}:// TODO deduplicate this helper later"); + let body = if i == 37 { + "fn sync_worker_v2() -> Result<()> { run_worker() }".to_string() + } else { + format!("ordinary worker helper number {i}") + }; + let _ = writeln!(s, "src/x.rs:{i}:{body}"); } - let out = compress(&s, None, false).expect("compresses").text; - let kept = out - .lines() - .filter(|line| line.starts_with("src/r.rs:")) - .count(); - assert_eq!(kept, TOP_K_PER_FILE, "redundant matches collapse:\n{out}"); + + let out = compress(&s, Some(r"sync_worker_v2\(")) + .expect("compresses") + .text; + assert!( - out.contains(&format!("[+{} more match(es) in src/r.rs]", 50 - kept)), - "{out}" + out.contains("fn sync_worker_v2()"), + "regex-style query identifier was not ranked in:\n{out}" ); } #[test] - fn adaptive_k_keeps_more_for_diverse_matches() { - const WORDS: [&str; 50] = [ - "database", - "migration", - "login", - "credential", - "cache", - "eviction", - "worker", - "heartbeat", - "handshake", - "cipher", - "latency", - "checkpoint", - "rollout", - "cohort", - "webhook", - "signature", - "resolver", - "nameserver", - "kernel", - "container", - "exporter", - "histogram", - "backlog", - "consumer", - "certificate", - "renewal", - "replica", - "election", - "limiter", - "burst", - "tokenizer", - "analyzer", - "cookie", - "rotation", - "saturation", - "queueing", - "multipart", - "upload", - "deadline", - "budget", - "ledger", - "quorum", - "snapshot", - "compaction", - "gossip", - "failover", - "throttle", - "warmup", - "vacuum", - "sharding", - ]; - let mut s = String::new(); - for i in 0..50 { - let _ = writeln!( - s, - "src/d.rs:{i}:{} {} {} {}", - WORDS[i], - WORDS[(i + 13) % 50], - WORDS[(i * 7 + 3) % 50], - WORDS[(i * 11 + 5) % 50], - ); - } - let out = compress(&s, None, false).expect("compresses").text; - let kept = out - .lines() - .filter(|line| line.starts_with("src/d.rs:")) - .count(); - assert_eq!(kept, MAX_K_PER_FILE, "diverse matches keep more:\n{out}"); + fn search_transform_requires_retained_ccr() { + let input = big_results(); + let pipeline_input = PipelineInput { + content: &input, + original_content: &input, + content_kind: ContentKind::Search, + original_bytes: input.len(), + }; + let transform = SearchTransform::new().with_query("compute_long_name_7"); + + let rejecting_store = MemoryCcrStore::new(1, 1); + assert!(transform.apply(&pipeline_input, &rejecting_store).is_none()); + + let store = MemoryCcrStore::default(); + let out = transform + .apply(&pipeline_input, &store) + .expect("retained search output"); + + assert_eq!(out.kind(), CompressorKind::Search); + assert!(out.text().contains("compute_long_name_7"), "{}", out.text()); + assert_eq!(store.get(out.token()).as_deref(), Some(input.as_str())); + } + + #[test] + fn search_transform_skips_non_search_input() { + let input = PipelineInput { + content: "plain text", + original_content: "plain text", + content_kind: ContentKind::PlainText, + original_bytes: "plain text".len(), + }; + let transform = SearchTransform::new(); + let store = MemoryCcrStore::default(); + + assert_eq!(transform.estimate_bloat(&input), 0.0); + assert!(transform.apply(&input, &store).is_none()); } #[test] fn small_result_set_passes_through() { let s = "a.rs:1:hit\nb.rs:2:hit\n"; - assert!(compress(s, None, false).is_none()); + assert!(compress(s, None).is_none()); } #[test] - fn context_lines_are_tallied_not_silently_dropped() { - // rg -C style output: context lines use `path-NN-body` and `--` - // separators, which don't parse as matches. - let mut s = String::new(); - for i in 0..50 { - let _ = writeln!(s, "src/a.rs:{}:let matched_line_{i} = f();", i * 4 + 2); - let _ = writeln!(s, "src/a.rs-{}-context line above", i * 4 + 1); - let _ = writeln!(s, "src/a.rs-{}-context line below", i * 4 + 3); - let _ = writeln!(s, "--"); - } - let out = compress(&s, None, false).expect("compresses").text; + fn ranked_search_exact_symbol_beats_path_and_density() { + let query = SearchReadQuery { + literal: Some("payment".to_string()), + symbols: vec!["settle_invoice".to_string()], + ..Default::default() + }; + let symbol = SearchReadCandidate { + path: "src/billing/worker.rs".into(), + matched_lines: vec![SearchReadLine::new(42, "fn settle_invoice() -> Result<()>")], + max_line: 100, + ..SearchReadCandidate::new("src/billing/worker.rs") + }; + let path_only = SearchReadCandidate { + path: "src/payment/readme.md".into(), + matched_lines: vec![SearchReadLine::new(1, "general overview")], + max_line: 10, + ..SearchReadCandidate::new("src/payment/readme.md") + }; + let density = SearchReadCandidate { + path: "src/other.rs".into(), + matched_lines: vec![ + SearchReadLine::new(10, "payment retry"), + SearchReadLine::new(11, "payment queue"), + ], + max_line: 20, + ..SearchReadCandidate::new("src/other.rs") + }; + + let symbol_score = rank_search_read_candidate(&symbol, &query); + let path_score = rank_search_read_candidate(&path_only, &query); + let density_score = rank_search_read_candidate(&density, &query); + + assert!(symbol_score > path_score, "{symbol_score} <= {path_score}"); assert!( - out.contains("context/summary line(s) omitted"), - "omission tally missing:\n{out}" + path_score > density_score, + "{path_score} <= {density_score}" ); } #[test] - fn more_matches_token_retrieves_omitted_matches() { - use crate::cache; - let input = big_results(); - let out = compress(&input, None, true).expect("compresses").text; - let tokens = cache::parse_markers(&out); - assert!(!tokens.is_empty(), "tallies should carry tokens: {out}"); - let block = cache::retrieve(&tokens[0]).expect("stored"); + fn ranked_search_deprioritizes_vendor_and_generated_paths() { + let query = SearchReadQuery { + symbols: vec!["hydrateRoot".to_string()], + ..Default::default() + }; + let first_party = SearchReadCandidate { + path: "src/app/root.tsx".into(), + matched_lines: vec![SearchReadLine::new(8, "export function hydrateRoot() {}")], + max_line: 20, + ..SearchReadCandidate::new("src/app/root.tsx") + }; + let vendor = SearchReadCandidate { + path: "node_modules/pkg/root.tsx".into(), + matched_lines: vec![SearchReadLine::new(8, "export function hydrateRoot() {}")], + max_line: 20, + vendor: true, + ..SearchReadCandidate::new("node_modules/pkg/root.tsx") + }; + assert!( - block.contains("src/a.rs:"), - "omitted matches for a.rs: {block}" + rank_search_read_candidate(&first_party, &query) + > rank_search_read_candidate(&vendor, &query) + ); + } + + #[test] + fn ranked_search_can_honor_explicit_vendor_scope() { + let query = SearchReadQuery { + symbols: vec!["hydrateRoot".to_string()], + penalize_vendor: false, + ..Default::default() + }; + let first_party = SearchReadCandidate { + path: "src/app/root.tsx".into(), + matched_lines: vec![SearchReadLine::new(8, "export function hydrateRoot() {}")], + max_line: 20, + ..SearchReadCandidate::new("src/app/root.tsx") + }; + let vendor = SearchReadCandidate { + path: "vendor/pkg/root.tsx".into(), + matched_lines: vec![SearchReadLine::new(8, "export function hydrateRoot() {}")], + max_line: 20, + vendor: true, + ..SearchReadCandidate::new("vendor/pkg/root.tsx") + }; + + assert_eq!( + rank_search_read_candidate(&first_party, &query), + rank_search_read_candidate(&vendor, &query) + ); + } + + #[test] + fn snippet_windows_merge_and_report_omitted_matches() { + let selected = select_snippet_windows(&[10, 11, 20, 50], 2, 100, 2); + + assert_eq!( + selected.windows, + vec![LineRange::new(8, 13), LineRange::new(18, 22)] ); - assert!(!block.contains("src/b.rs:"), "must not mix files: {block}"); + assert_eq!(selected.omitted_matches, 1); } } diff --git a/src/compressors/text.rs b/src/compressors/text.rs new file mode 100644 index 0000000..8e46bf4 --- /dev/null +++ b/src/compressors/text.rs @@ -0,0 +1,946 @@ +//! Plain-text compressor. +//! +//! ML compression is tried first when enabled. When it is unavailable or +//! disabled, TextCrusher provides a deterministic extractive fallback: it keeps +//! verbatim salient/query-relevant spans and suppresses near-duplicates. + +use async_trait::async_trait; +use std::collections::HashSet; + +use super::Compressor; +use crate::cache::CcrStore; +use crate::pipeline::{OffloadOutput, OffloadTransform, PipelineInput, estimate_bloat}; +use crate::relevance::Bm25Corpus; +use crate::types::{CompressInput, CompressOptions, CompressOutput, CompressorKind, ContentKind}; + +pub const MIN_SEGMENTS: usize = 12; +pub const TARGET_RATIO: f32 = 0.40; +const MIN_TARGET_CHARS: usize = 800; + +pub struct TextCompressor; + +/// Typed offload transform for deterministic extractive TextCrusher. +#[derive(Debug, Clone)] +pub struct TextCrusherTransform { + options: CompressOptions, + query: Option, +} + +impl TextCrusherTransform { + pub fn new(options: CompressOptions) -> Self { + Self { + options, + query: None, + } + } + + pub fn with_query(mut self, query: impl Into) -> Self { + self.query = Some(query.into()); + self + } + + pub fn options(&self) -> &CompressOptions { + &self.options + } + + pub fn query(&self) -> Option<&str> { + self.query.as_deref() + } +} + +impl Default for TextCrusherTransform { + fn default() -> Self { + Self::new(CompressOptions::default()) + } +} + +impl OffloadTransform for TextCrusherTransform { + fn name(&self) -> &'static str { + "textcrusher" + } + + fn estimate_bloat(&self, input: &PipelineInput<'_>) -> f32 { + if input.content_kind != ContentKind::PlainText { + return 0.0; + } + let score = f32::from(estimate_bloat(input.content, input.content_kind).score) / 100.0; + if self + .query + .as_deref() + .is_some_and(|query| !query.trim().is_empty()) + { + score.max(0.1) + } else { + score + } + } + + fn apply(&self, input: &PipelineInput<'_>, store: &dyn CcrStore) -> Option { + if input.content_kind != ContentKind::PlainText { + return None; + } + let compacted = compress_textcrusher(input.content, self.query(), &self.options)?; + OffloadOutput::from_retained_put( + compacted.text, + CompressorKind::TextCrusher, + store.put(input.original_content), + ) + } +} + +#[async_trait] +impl Compressor for TextCompressor { + fn kind(&self) -> CompressorKind { + CompressorKind::TextCrusher + } + + async fn compress( + &self, + input: &CompressInput<'_>, + opts: &CompressOptions, + ) -> Option { + if let Some(out) = compress_ml_with_tag_protection(input.content, opts).await { + return Some(out); + } + + compress_textcrusher(input.content, input.hint.query.as_deref(), opts) + } +} + +#[derive(Debug, Clone)] +struct Segment { + index: usize, + text: String, + terms: Vec, + score: f32, + salience: f32, +} + +pub async fn compress_ml_with_tag_protection( + content: &str, + opts: &CompressOptions, +) -> Option { + if !opts.ml_text_enabled { + return None; + } + + let protected = protect_custom_tags(content); + match crate::ml::compress(&protected.text, opts).await { + Ok(Some(text)) => { + let text = protected.restore(&text)?; + if text.len() < content.len() { + Some(CompressOutput::lossy(text, CompressorKind::MlText)) + } else { + None + } + } + Ok(_) => None, + Err(e) => { + log::debug!("[tokenjuice][ml] unavailable, falling back: {e:#}"); + None + } + } +} + +pub fn compress_textcrusher( + content: &str, + query: Option<&str>, + opts: &CompressOptions, +) -> Option { + let mut segments = split_segments(content); + if segments.len() < MIN_SEGMENTS { + return None; + } + + let duplicate_pressure = duplicate_pressure(&segments); + let query = query.filter(|q| !q.trim().is_empty()); + let bm25 = query.map(|_| { + Bm25Corpus::from_tokenized( + segments + .iter() + .map(|segment| segment.terms.clone()) + .collect(), + ) + }); + + let segment_count = segments.len(); + for segment in &mut segments { + segment.salience = salience_score(&segment.text); + let recency = (segment.index + 1) as f32 / segment_count as f32; + let query_score = match (query, bm25.as_ref()) { + (Some(q), Some(corpus)) => corpus.score(q, segment.index).min(4.0) / 4.0, + _ => 0.0, + }; + segment.score = segment.salience * 2.0 + query_score * 2.5 + recency * 0.35; + } + + let has_signal = query.is_some() + || duplicate_pressure >= 4 + || segments.iter().any(|segment| segment.salience >= 0.5); + if !has_signal { + return None; + } + + let ratio_target = (content.len() as f32 * TARGET_RATIO) as usize; + let target_chars = opts + .max_inline_chars + .unwrap_or(ratio_target) + .min(ratio_target.max(MIN_TARGET_CHARS)) + .min(content.len().saturating_sub(1)); + if target_chars == 0 { + return None; + } + + let mut ranked: Vec = (0..segments.len()).collect(); + ranked.sort_by(|&a, &b| { + segments[b] + .score + .partial_cmp(&segments[a].score) + .unwrap_or(std::cmp::Ordering::Equal) + .then_with(|| a.cmp(&b)) + }); + + let mut selected = Vec::new(); + let mut selected_terms: Vec> = Vec::new(); + let mut used_chars = 0usize; + + for index in ranked { + let segment = &segments[index]; + if used_chars + segment.text.len() + 1 > target_chars && !selected.is_empty() { + continue; + } + let terms: HashSet = segment.terms.iter().cloned().collect(); + if selected_terms + .iter() + .any(|existing| jaccard(existing, &terms) >= 0.82) + { + continue; + } + used_chars += segment.text.len() + 1; + selected.push(index); + selected_terms.push(terms); + } + + if selected.is_empty() || selected.len() == segments.len() { + return None; + } + + selected.sort_unstable(); + let out = selected + .into_iter() + .map(|index| segments[index].text.as_str()) + .collect::>() + .join("\n"); + + if out.len() >= content.len() { + return None; + } + Some(CompressOutput::lossy(out, CompressorKind::TextCrusher)) +} + +#[derive(Debug, Clone, PartialEq, Eq)] +struct ProtectedText { + text: String, + replacements: Vec, +} + +impl ProtectedText { + fn restore(&self, candidate: &str) -> Option { + for replacement in &self.replacements { + if !candidate.contains(&replacement.placeholder) { + return None; + } + } + + let mut restored = candidate.to_string(); + for replacement in &self.replacements { + restored = restored.replace(&replacement.placeholder, &replacement.original); + } + Some(restored) + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +struct TagReplacement { + placeholder: String, + original: String, +} + +fn protect_custom_tags(content: &str) -> ProtectedText { + let ranges = custom_tag_ranges(content); + if ranges.is_empty() { + return ProtectedText { + text: content.to_string(), + replacements: Vec::new(), + }; + } + + let prefix = safe_placeholder_prefix(content); + let mut text = String::with_capacity(content.len()); + let mut replacements = Vec::new(); + let mut cursor = 0usize; + + for (i, range) in ranges.into_iter().enumerate() { + if range.start < cursor { + continue; + } + text.push_str(&content[cursor..range.start]); + let placeholder = format!("{prefix}{i}__"); + text.push_str(&placeholder); + replacements.push(TagReplacement { + placeholder, + original: content[range.start..range.end].to_string(), + }); + cursor = range.end; + } + text.push_str(&content[cursor..]); + + ProtectedText { text, replacements } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +struct ByteRange { + start: usize, + end: usize, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +struct OpenTag { + name: String, + start: usize, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum TagKind { + Open, + Close, + SelfClosing, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +struct ParsedTag { + range: ByteRange, + name: String, + kind: TagKind, + custom: bool, +} + +fn custom_tag_ranges(content: &str) -> Vec { + let mut ranges = Vec::new(); + let mut stack: Vec = Vec::new(); + let mut cursor = 0usize; + + while let Some(relative) = content[cursor..].find('<') { + let start = cursor + relative; + let Some(tag) = parse_tag_at(content, start) else { + cursor = start + 1; + continue; + }; + cursor = tag.range.end; + + if !tag.custom { + continue; + } + + match tag.kind { + TagKind::SelfClosing => ranges.push(tag.range), + TagKind::Open => stack.push(OpenTag { + name: tag.name, + start: tag.range.start, + }), + TagKind::Close => { + if let Some(pos) = stack.iter().rposition(|open| open.name == tag.name) { + let open = stack.remove(pos); + stack.truncate(pos); + ranges.push(ByteRange { + start: open.start, + end: tag.range.end, + }); + } + } + } + } + + ranges.sort_by(|a, b| a.start.cmp(&b.start).then_with(|| b.end.cmp(&a.end))); + let mut non_overlapping = Vec::new(); + let mut covered_end = 0usize; + for range in ranges { + if range.start >= covered_end { + covered_end = range.end; + non_overlapping.push(range); + } + } + non_overlapping +} + +fn parse_tag_at(content: &str, start: usize) -> Option { + let tail = content.get(start..)?; + if !tail.starts_with('<') + || tail.starts_with(""; +const TINYJUICE_MANAGED_END: &str = ""; + +fn main() -> ExitCode { + match run(std::env::args().skip(1).collect()) { + Ok(()) => ExitCode::SUCCESS, + Err(error) => { + let _ = writeln!(io::stderr(), "{error}"); + ExitCode::from(2) + } + } +} + +fn run(args: Vec) -> Result<(), String> { + let Some(command) = args.first().map(String::as_str) else { + print_usage(); + return Err("missing command".to_owned()); + }; + + match command { + "reduce" => run_reduce(&args[1..]), + "reduce-json" => run_reduce_json(&args[1..]), + "verify" => run_verify(&args[1..]), + "discover" => run_discover(&args[1..]), + "wrap" => run_wrap(&args[1..]), + "ls" => run_ls(&args[1..]), + "cat" => run_cat(&args[1..]), + "stats" => run_stats(&args[1..]), + "doctor" => run_doctor(&args[1..]), + "install" => run_install(&args[1..]), + "uninstall" => run_uninstall(&args[1..]), + "-h" | "--help" | "help" => { + print_usage(); + Ok(()) + } + other => { + print_usage(); + Err(format!("unknown command: {other}")) + } + } +} + +fn run_ls(args: &[String]) -> Result<(), String> { + if args.iter().any(|arg| arg == "-h" || arg == "--help") { + print_ls_usage(); + return Ok(()); + } + let store_dir = parse_store_dir_args("ls", args)?; + let inventory = read_ccr_store_inventory(&store_dir)?; + for entry in inventory.entries { + println!("{}", entry.token); + } + Ok(()) +} + +fn run_stats(args: &[String]) -> Result<(), String> { + let mut store_dir = None; + let mut pretty = false; + let mut i = 0; + + while i < args.len() { + match args[i].as_str() { + "--store-dir" => { + i += 1; + store_dir = Some(PathBuf::from( + args.get(i) + .ok_or_else(|| "--store-dir requires a value".to_owned())?, + )); + } + "--pretty" => pretty = true, + "-h" | "--help" => { + print_stats_usage(); + return Ok(()); + } + value => return Err(format!("unknown stats option: {value}")), + } + i += 1; + } + + let store_dir = store_dir.ok_or_else(|| "stats requires --store-dir".to_owned())?; + let inventory = read_ccr_store_inventory(&store_dir)?; + let bytes = inventory + .entries + .iter() + .map(|entry| entry.bytes) + .sum::(); + let stats = serde_json::json!({ + "storeDir": store_dir.to_string_lossy(), + "tokens": inventory.entries.len(), + "bytes": bytes, + "ignoredEntries": inventory.ignored_entries, + }); + if pretty { + println!( + "{}", + serde_json::to_string_pretty(&stats) + .map_err(|error| format!("failed to serialize stats: {error}"))? + ); + } else { + println!( + "{}", + serde_json::to_string(&stats) + .map_err(|error| format!("failed to serialize stats: {error}"))? + ); + } + Ok(()) +} + +fn run_doctor(args: &[String]) -> Result<(), String> { + let mut target = None; + let mut store_dir = None; + let mut fixtures_dir = PathBuf::from("tests/fixtures"); + let mut codex_home = None; + let mut openhuman_root = None; + let mut check_fixtures = false; + let mut pretty = false; + let mut i = 0; + + while i < args.len() { + match args[i].as_str() { + "--store-dir" => { + i += 1; + store_dir = Some(PathBuf::from( + args.get(i) + .ok_or_else(|| "--store-dir requires a value".to_owned())?, + )); + } + "--codex-home" => { + i += 1; + codex_home = Some(PathBuf::from( + args.get(i) + .ok_or_else(|| "--codex-home requires a value".to_owned())?, + )); + } + "--openhuman-root" => { + i += 1; + openhuman_root = + Some(PathBuf::from(args.get(i).ok_or_else(|| { + "--openhuman-root requires a value".to_owned() + })?)); + } + "--fixtures" => { + check_fixtures = true; + if let Some(next) = args.get(i + 1) + && !next.starts_with('-') + { + i += 1; + fixtures_dir = PathBuf::from(next); + } + } + "--no-fixtures" => check_fixtures = false, + "--pretty" => pretty = true, + "-h" | "--help" => { + print_doctor_usage(); + return Ok(()); + } + value if value.starts_with('-') => { + return Err(format!("unknown doctor option: {value}")); + } + value => { + if target.replace(value.to_owned()).is_some() { + return Err("doctor accepts at most one host target".to_owned()); + } + } + } + i += 1; + } + + if let Some(target) = target { + return run_host_doctor(&target, pretty, codex_home, openhuman_root); + } + + let mut status = "disabled"; + let mut checks = Vec::new(); + + let rules_report = verify_rules(&LoadRuleOptions { + exclude_user: true, + exclude_project: true, + ..Default::default() + }); + let rules_status = if !rules_report.parse_errors.is_empty() + || !rules_report.duplicate_ids.is_empty() + { + "broken" + } else if !rules_report.invalid_regexes.is_empty() || !rules_report.shadowed_rules.is_empty() { + "warn" + } else { + "ok" + }; + status = merge_doctor_status(status, rules_status); + checks.push(serde_json::json!({ + "name": "rules", + "status": rules_status, + "descriptors": rules_report.descriptors_seen, + "valid": rules_report.valid_rules, + "final": rules_report.final_rules, + "parseErrors": rules_report.parse_errors.len(), + "invalidRegexes": rules_report.invalid_regexes.len(), + "duplicateIds": rules_report.duplicate_ids.len(), + "shadowed": rules_report.shadowed_rules.len(), + })); + + if check_fixtures { + let rules = load_builtin_rules(); + let fixture_report = verify_rule_fixtures(&fixtures_dir, &rules); + let fixture_status = if fixture_report.is_clean() { + "ok" + } else { + "broken" + }; + status = merge_doctor_status(status, fixture_status); + checks.push(serde_json::json!({ + "name": "fixtures", + "status": fixture_status, + "dir": fixtures_dir.to_string_lossy(), + "seen": fixture_report.fixtures_seen, + "passed": fixture_report.passed, + "parseErrors": fixture_report.parse_errors.len(), + "failures": fixture_report.failures.len(), + })); + } else { + checks.push(serde_json::json!({ + "name": "fixtures", + "status": "disabled", + })); + } + + match store_dir { + Some(store_dir) => match read_ccr_store_inventory(&store_dir) { + Ok(inventory) => { + let bytes = inventory + .entries + .iter() + .map(|entry| entry.bytes) + .sum::(); + status = merge_doctor_status(status, "ok"); + checks.push(serde_json::json!({ + "name": "ccrDiskStore", + "status": "ok", + "storeDir": store_dir.to_string_lossy(), + "tokens": inventory.entries.len(), + "bytes": bytes, + "ignoredEntries": inventory.ignored_entries, + })); + } + Err(error) => { + status = merge_doctor_status(status, "broken"); + checks.push(serde_json::json!({ + "name": "ccrDiskStore", + "status": "broken", + "storeDir": store_dir.to_string_lossy(), + "error": error, + })); + } + }, + None => checks.push(serde_json::json!({ + "name": "ccrDiskStore", + "status": "disabled", + })), + } + + let report = serde_json::json!({ + "status": status, + "checks": checks, + }); + if pretty { + println!( + "{}", + serde_json::to_string_pretty(&report) + .map_err(|error| format!("failed to serialize doctor report: {error}"))? + ); + } else { + println!( + "{}", + serde_json::to_string(&report) + .map_err(|error| format!("failed to serialize doctor report: {error}"))? + ); + } + + if status == "broken" { + Err("doctor found broken checks".to_owned()) + } else { + Ok(()) + } +} + +fn run_host_doctor( + target: &str, + pretty: bool, + codex_home: Option, + openhuman_root: Option, +) -> Result<(), String> { + let checks = match target { + "codex" => vec![doctor_codex(codex_home)], + "openhuman" => vec![doctor_openhuman(openhuman_root)], + "hooks" => vec![doctor_codex(codex_home), doctor_openhuman(openhuman_root)], + other => return Err(format!("unknown doctor host target: {other}")), + }; + let status = checks + .iter() + .filter_map(|check| check.get("status").and_then(serde_json::Value::as_str)) + .fold("disabled", merge_doctor_status); + let report = serde_json::json!({ + "status": status, + "checks": checks, + }); + if pretty { + println!( + "{}", + serde_json::to_string_pretty(&report) + .map_err(|error| format!("failed to serialize doctor report: {error}"))? + ); + } else { + println!( + "{}", + serde_json::to_string(&report) + .map_err(|error| format!("failed to serialize doctor report: {error}"))? + ); + } + + if status == "broken" { + Err("doctor found broken host checks".to_owned()) + } else { + Ok(()) + } +} + +fn run_install(args: &[String]) -> Result<(), String> { + if args.iter().any(|arg| arg == "-h" || arg == "--help") { + print_install_usage(); + return Ok(()); + } + let install = parse_host_mutation_args("install", args, true)?; + match install.host.as_str() { + "codex" => { + let report = install_codex(install.codex_home, install.local_binary)?; + println!( + "{}", + serde_json::to_string(&report) + .map_err(|error| format!("failed to serialize install report: {error}"))? + ); + Ok(()) + } + other => Err(format!("unknown install host: {other}")), + } +} + +fn run_uninstall(args: &[String]) -> Result<(), String> { + if args.iter().any(|arg| arg == "-h" || arg == "--help") { + print_uninstall_usage(); + return Ok(()); + } + let uninstall = parse_host_mutation_args("uninstall", args, false)?; + match uninstall.host.as_str() { + "codex" => { + let report = uninstall_codex(uninstall.codex_home)?; + println!( + "{}", + serde_json::to_string(&report) + .map_err(|error| format!("failed to serialize uninstall report: {error}"))? + ); + Ok(()) + } + other => Err(format!("unknown uninstall host: {other}")), + } +} + +struct HostMutationArgs { + host: String, + codex_home: Option, + local_binary: Option, +} + +fn parse_host_mutation_args( + command: &str, + args: &[String], + allow_local: bool, +) -> Result { + let mut host = None; + let mut codex_home = None; + let mut local_binary = None; + let mut i = 0; + + while i < args.len() { + match args[i].as_str() { + "--codex-home" => { + i += 1; + codex_home = Some(PathBuf::from( + args.get(i) + .ok_or_else(|| "--codex-home requires a value".to_owned())?, + )); + } + "--local" if allow_local => { + i += 1; + local_binary = + Some(PathBuf::from(args.get(i).ok_or_else(|| { + "--local requires a binary path".to_owned() + })?)); + } + value if value.starts_with('-') => { + return Err(format!("unknown {command} option: {value}")); + } + value => { + if host.replace(value.to_owned()).is_some() { + return Err(format!("{command} accepts exactly one host")); + } + } + } + i += 1; + } + + Ok(HostMutationArgs { + host: host.ok_or_else(|| format!("{command} requires a host"))?, + codex_home, + local_binary, + }) +} + +fn install_codex( + codex_home: Option, + local_binary: Option, +) -> Result { + let root = codex_home.unwrap_or_else(default_codex_home); + let instruction_path = codex_instruction_path(&root); + let binary = local_binary + .map(|path| path.to_string_lossy().into_owned()) + .unwrap_or_else(|| "tinyjuice".to_owned()); + let block = managed_codex_block(&binary); + let original = match fs::read_to_string(&instruction_path) { + Ok(content) => content, + Err(error) if error.kind() == io::ErrorKind::NotFound => String::new(), + Err(error) => { + return Err(format!( + "failed to read {}: {error}", + instruction_path.display() + )); + } + }; + let updated = replace_or_insert_managed_block(&original, &block); + let changed = updated != original; + if changed { + write_managed_file(&instruction_path, &original, &updated)?; + } + Ok(serde_json::json!({ + "host": "codex", + "status": "ok", + "changed": changed, + "configPath": instruction_path.to_string_lossy(), + "command": format!("{binary} reduce-json"), + })) +} + +fn uninstall_codex(codex_home: Option) -> Result { + let root = codex_home.unwrap_or_else(default_codex_home); + let instruction_path = codex_instruction_path(&root); + let original = match fs::read_to_string(&instruction_path) { + Ok(content) => content, + Err(error) if error.kind() == io::ErrorKind::NotFound => { + return Ok(serde_json::json!({ + "host": "codex", + "status": "ok", + "changed": false, + "configPath": instruction_path.to_string_lossy(), + })); + } + Err(error) => { + return Err(format!( + "failed to read {}: {error}", + instruction_path.display() + )); + } + }; + let updated = remove_managed_block(&original); + let changed = updated != original; + if changed { + write_managed_file(&instruction_path, &original, &updated)?; + } + Ok(serde_json::json!({ + "host": "codex", + "status": "ok", + "changed": changed, + "configPath": instruction_path.to_string_lossy(), + })) +} + +fn doctor_codex(codex_home: Option) -> serde_json::Value { + let root = codex_home.unwrap_or_else(default_codex_home); + let instruction_path = codex_instruction_path(&root); + let repair_command = "tinyjuice install codex".to_owned(); + let expected_command = "tinyjuice reduce-json".to_owned(); + + let content = match fs::read_to_string(&instruction_path) { + Ok(content) => content, + Err(error) if error.kind() == io::ErrorKind::NotFound => { + return serde_json::json!({ + "name": "codex", + "status": "disabled", + "configPath": instruction_path.to_string_lossy(), + "expectedCommand": expected_command, + "repairCommand": repair_command, + }); + } + Err(error) => { + return serde_json::json!({ + "name": "codex", + "status": "broken", + "configPath": instruction_path.to_string_lossy(), + "expectedCommand": expected_command, + "repairCommand": repair_command, + "error": format!("failed to read Codex TinyJuice instructions: {error}"), + }); + } + }; + + let Some(block) = managed_block(&content) else { + return serde_json::json!({ + "name": "codex", + "status": "disabled", + "configPath": instruction_path.to_string_lossy(), + "expectedCommand": expected_command, + "repairCommand": repair_command, + }); + }; + let detected_command = managed_command(block); + let status = match detected_command.as_deref().and_then(command_binary_status) { + Some(false) => "broken", + _ => "ok", + }; + let mut check = serde_json::json!({ + "name": "codex", + "status": status, + "configPath": instruction_path.to_string_lossy(), + "expectedCommand": expected_command, + "detectedCommand": detected_command, + "repairCommand": repair_command, + }); + if status == "broken" { + check["error"] = + serde_json::Value::String("managed command points at a missing executable".to_owned()); + } + check +} + +fn doctor_openhuman(openhuman_root: Option) -> serde_json::Value { + let root = openhuman_root.unwrap_or_else(default_openhuman_root); + let adapter_path = root + .join("src") + .join("openhuman") + .join("tokenjuice") + .join("mod.rs"); + if adapter_path.is_file() { + serde_json::json!({ + "name": "openhuman", + "status": "ok", + "configPath": adapter_path.to_string_lossy(), + "expectedCommand": "direct Rust crate integration", + "detectedCommand": "src/openhuman/tokenjuice", + "repairCommand": "tinyjuice doctor openhuman --openhuman-root PATH", + }) + } else { + serde_json::json!({ + "name": "openhuman", + "status": "disabled", + "configPath": adapter_path.to_string_lossy(), + "expectedCommand": "direct Rust crate integration", + "repairCommand": "tinyjuice doctor openhuman --openhuman-root PATH", + }) + } +} + +fn default_codex_home() -> PathBuf { + std::env::var_os("CODEX_HOME") + .map(PathBuf::from) + .or_else(|| std::env::var_os("HOME").map(|home| PathBuf::from(home).join(".codex"))) + .unwrap_or_else(|| PathBuf::from(".codex")) +} + +fn default_openhuman_root() -> PathBuf { + PathBuf::from("../openhuman-4") +} + +fn codex_instruction_path(root: &Path) -> PathBuf { + root.join("instructions").join("tinyjuice.md") +} + +fn managed_codex_block(binary: &str) -> String { + format!( + "{TINYJUICE_MANAGED_BEGIN}\n\ + # TinyJuice\n\ + Command: {binary} reduce-json\n\ + When tool output is too large for context, prefer passing a structured payload to the command above or using `tinyjuice wrap -- ` when no post-tool hook is available.\n\ + {TINYJUICE_MANAGED_END}" + ) +} + +fn managed_block(content: &str) -> Option<&str> { + let start = content.find(TINYJUICE_MANAGED_BEGIN)? + TINYJUICE_MANAGED_BEGIN.len(); + let end = content[start..].find(TINYJUICE_MANAGED_END)? + start; + Some(content[start..end].trim()) +} + +fn replace_or_insert_managed_block(content: &str, block: &str) -> String { + if let Some((start, end)) = managed_block_bounds(content) { + let mut out = String::with_capacity(content.len() + block.len()); + out.push_str(&content[..start]); + out.push_str(block); + out.push_str(&content[end..]); + out + } else if content.trim().is_empty() { + format!("{block}\n") + } else { + format!("{}\n\n{block}\n", content.trim_end()) + } +} + +fn remove_managed_block(content: &str) -> String { + if let Some((start, end)) = managed_block_bounds(content) { + let mut out = String::with_capacity(content.len()); + out.push_str(content[..start].trim_end()); + out.push_str(content[end..].trim_start_matches('\n')); + if !out.is_empty() && !out.ends_with('\n') { + out.push('\n'); + } + out + } else { + content.to_owned() + } +} + +fn managed_block_bounds(content: &str) -> Option<(usize, usize)> { + let start = content.find(TINYJUICE_MANAGED_BEGIN)?; + let end = content[start..].find(TINYJUICE_MANAGED_END)? + start + TINYJUICE_MANAGED_END.len(); + Some((start, end)) +} + +fn write_managed_file(path: &Path, original: &str, updated: &str) -> Result<(), String> { + let parent = path + .parent() + .ok_or_else(|| format!("{} has no parent directory", path.display()))?; + fs::create_dir_all(parent) + .map_err(|error| format!("failed to create {}: {error}", parent.display()))?; + if path.exists() && !original.is_empty() { + let backup = path.with_extension("md.bak"); + if !backup.exists() { + fs::write(&backup, original) + .map_err(|error| format!("failed to write backup {}: {error}", backup.display()))?; + } + } + let tmp = path.with_extension("md.tmp"); + fs::write(&tmp, updated) + .map_err(|error| format!("failed to write temp file {}: {error}", tmp.display()))?; + fs::rename(&tmp, path).map_err(|error| format!("failed to replace {}: {error}", path.display())) +} + +fn managed_command(block: &str) -> Option { + block.lines().find_map(|line| { + line.trim() + .strip_prefix("Command:") + .map(str::trim) + .filter(|command| !command.is_empty()) + .map(str::to_owned) + }) +} + +fn command_binary_status(command: &str) -> Option { + let binary = command.split_whitespace().next()?; + if binary.contains(std::path::MAIN_SEPARATOR) || Path::new(binary).is_absolute() { + Some(Path::new(binary).is_file()) + } else { + None + } +} + +fn run_cat(args: &[String]) -> Result<(), String> { + let mut store_dir = None; + let mut token = None; + let mut range = None; + let mut i = 0; + + while i < args.len() { + match args[i].as_str() { + "--store-dir" => { + i += 1; + store_dir = Some(PathBuf::from( + args.get(i) + .ok_or_else(|| "--store-dir requires a value".to_owned())?, + )); + } + "--lines" => { + if range.is_some() { + return Err("cat accepts at most one range flag".to_owned()); + } + i += 1; + range = Some(( + RangeUnit::Lines, + parse_range_arg( + args.get(i) + .ok_or_else(|| "--lines requires START:END".to_owned())?, + "--lines", + )?, + )); + } + "--bytes" => { + if range.is_some() { + return Err("cat accepts at most one range flag".to_owned()); + } + i += 1; + range = Some(( + RangeUnit::Bytes, + parse_range_arg( + args.get(i) + .ok_or_else(|| "--bytes requires START:END".to_owned())?, + "--bytes", + )?, + )); + } + "-h" | "--help" => { + print_cat_usage(); + return Ok(()); + } + value if value.starts_with('-') => return Err(format!("unknown cat option: {value}")), + value => { + if token.replace(value.to_owned()).is_some() { + return Err("cat accepts exactly one token".to_owned()); + } + } + } + i += 1; + } + + let store_dir = store_dir.ok_or_else(|| "cat requires --store-dir".to_owned())?; + let token = token.ok_or_else(|| "cat requires a token".to_owned())?; + if !is_cli_ccr_token(&token) { + return Err("cat token must be a 32-character hex CCR token".to_owned()); + } + + tinyjuice::cache::enable_disk_tier(store_dir); + let content = match range { + Some((unit, (start, end))) => tinyjuice::cache::retrieve_range(&token, start, end, unit), + None => tinyjuice::cache::retrieve(&token), + } + .ok_or_else(|| format!("token not found: {token}"))?; + print!("{content}"); + Ok(()) +} + +fn parse_store_dir_args(command: &str, args: &[String]) -> Result { + let mut store_dir = None; + let mut i = 0; + + while i < args.len() { + match args[i].as_str() { + "--store-dir" => { + i += 1; + store_dir = Some(PathBuf::from( + args.get(i) + .ok_or_else(|| "--store-dir requires a value".to_owned())?, + )); + } + value => return Err(format!("unknown {command} option: {value}")), + } + i += 1; + } + + store_dir.ok_or_else(|| format!("{command} requires --store-dir")) +} + +fn parse_range_arg(raw: &str, flag: &str) -> Result<(usize, usize), String> { + let (start, end) = raw + .split_once(':') + .ok_or_else(|| format!("{flag} expects START:END"))?; + let start = start + .parse::() + .map_err(|_| format!("{flag} start must be a non-negative integer"))?; + let end = end + .parse::() + .map_err(|_| format!("{flag} end must be a non-negative integer"))?; + if end < start { + return Err(format!("{flag} end must be greater than or equal to start")); + } + Ok((start, end)) +} + +fn is_cli_ccr_token(token: &str) -> bool { + token.len() == 32 && token.bytes().all(|byte| byte.is_ascii_hexdigit()) +} + +struct CcrStoreInventory { + entries: Vec, + ignored_entries: usize, +} + +struct CcrStoreEntry { + token: String, + bytes: u64, +} + +fn read_ccr_store_inventory(store_dir: &Path) -> Result { + let mut entries = Vec::new(); + let mut ignored_entries = 0; + + for entry in fs::read_dir(store_dir) + .map_err(|error| format!("failed to read {}: {error}", store_dir.display()))? + { + let entry = entry.map_err(|error| format!("failed to read store entry: {error}"))?; + if !entry + .file_type() + .map_err(|error| format!("failed to inspect store entry: {error}"))? + .is_file() + { + ignored_entries += 1; + continue; + } + let name = entry.file_name().to_string_lossy().into_owned(); + if is_cli_ccr_token(&name) { + let bytes = entry + .metadata() + .map_err(|error| format!("failed to inspect store entry {name}: {error}"))? + .len(); + entries.push(CcrStoreEntry { token: name, bytes }); + } else { + ignored_entries += 1; + } + } + + entries.sort_by(|left, right| left.token.cmp(&right.token)); + Ok(CcrStoreInventory { + entries, + ignored_entries, + }) +} + +fn merge_doctor_status(current: &str, next: &str) -> &'static str { + match doctor_status_rank(current).max(doctor_status_rank(next)) { + 3 => "broken", + 2 => "warn", + 1 => "ok", + _ => "disabled", + } +} + +fn doctor_status_rank(status: &str) -> u8 { + match status { + "broken" => 3, + "warn" => 2, + "ok" => 1, + _ => 0, + } +} + +fn run_wrap(args: &[String]) -> Result<(), String> { + let mut tool_name = "exec".to_owned(); + let mut max_inline_chars = None; + let mut separator = None; + let mut i = 0; + + while i < args.len() { + match args[i].as_str() { + "--" => { + separator = Some(i); + break; + } + "--tool-name" => { + i += 1; + tool_name = args + .get(i) + .ok_or_else(|| "--tool-name requires a value".to_owned())? + .clone(); + } + "--max-inline-chars" => { + i += 1; + let value = args + .get(i) + .ok_or_else(|| "--max-inline-chars requires a value".to_owned())?; + max_inline_chars = Some( + value + .parse::() + .map_err(|_| "--max-inline-chars must be a positive integer".to_owned())?, + ); + } + "-h" | "--help" => { + print_wrap_usage(); + return Ok(()); + } + value => return Err(format!("unknown wrap option before --: {value}")), + } + i += 1; + } + + let Some(separator) = separator else { + return Err("wrap requires -- before the command".to_owned()); + }; + let command_args = &args[separator + 1..]; + let Some(program) = command_args.first() else { + return Err("wrap requires a command after --".to_owned()); + }; + + let output = Command::new(program) + .args(&command_args[1..]) + .output() + .map_err(|error| format!("failed to run {program}: {error}"))?; + let exit_code = output.status.code(); + let stdout = String::from_utf8_lossy(&output.stdout).into_owned(); + let stderr = String::from_utf8_lossy(&output.stderr).into_owned(); + let input = ToolExecutionInput { + tool_name, + command: Some(command_args.join(" ")), + argv: Some(command_args.to_vec()), + stdout: Some(stdout), + stderr: Some(stderr), + exit_code, + ..Default::default() + }; + let options = ReduceOptions { + max_inline_chars, + ..Default::default() + }; + let rules = load_builtin_rules(); + let result = reduce_execution_with_rules(input, &rules, &options); + print!("{}", result.inline_text); + + if output.status.success() { + Ok(()) + } else { + std::process::exit(exit_code.unwrap_or(1).clamp(1, 255)); + } +} + +fn run_discover(args: &[String]) -> Result<(), String> { + let mut pretty = false; + let mut path = None; + let mut i = 0; + + while i < args.len() { + match args[i].as_str() { + "--pretty" => pretty = true, + "-h" | "--help" => { + print_discover_usage(); + return Ok(()); + } + value if value.starts_with('-') && value != "-" => { + return Err(format!("unknown discover option: {value}")); + } + value => { + if path.replace(value.to_owned()).is_some() { + return Err("discover accepts at most one path".to_owned()); + } + } + } + i += 1; + } + + let payload = read_input(path.as_deref())?; + let inputs = parse_discovery_inputs(&payload)?; + let rules = load_builtin_rules(); + let report = discover_fallback_outputs(inputs, &rules); + if pretty { + println!( + "{}", + serde_json::to_string_pretty(&report) + .map_err(|error| format!("failed to serialize discovery report: {error}"))? + ); + } else { + println!( + "{}", + serde_json::to_string(&report) + .map_err(|error| format!("failed to serialize discovery report: {error}"))? + ); + } + Ok(()) +} + +fn parse_discovery_inputs(payload: &str) -> Result, String> { + let trimmed = payload.trim(); + if trimmed.is_empty() { + return Ok(Vec::new()); + } + + if trimmed.starts_with('[') { + return serde_json::from_str(trimmed) + .map_err(|error| format!("invalid discovery JSON array: {error}")); + } + + let mut inputs = Vec::new(); + for (line_index, line) in payload.lines().enumerate() { + let line = line.trim(); + if line.is_empty() { + continue; + } + let input = serde_json::from_str::(line) + .map_err(|error| format!("invalid discovery JSON line {}: {error}", line_index + 1))?; + inputs.push(input); + } + Ok(inputs) +} + +fn run_verify(args: &[String]) -> Result<(), String> { + let mut check_rules = false; + let mut check_fixtures = false; + let mut fixtures_dir = PathBuf::from("tests/fixtures"); + let mut i = 0; + + while i < args.len() { + match args[i].as_str() { + "--rules" => check_rules = true, + "--fixtures" => { + check_fixtures = true; + if let Some(next) = args.get(i + 1) + && !next.starts_with('-') + { + i += 1; + fixtures_dir = PathBuf::from(next); + } + } + "-h" | "--help" => { + print_verify_usage(); + return Ok(()); + } + value => return Err(format!("unknown verify option: {value}")), + } + i += 1; + } + + if !check_rules && !check_fixtures { + check_rules = true; + check_fixtures = true; + } + + let mut failed = false; + if check_rules { + let report = verify_rules(&LoadRuleOptions { + exclude_user: true, + exclude_project: true, + ..Default::default() + }); + println!( + "rules: descriptors={} valid={} final={} parse_errors={} invalid_regexes={} duplicate_ids={} shadowed={}", + report.descriptors_seen, + report.valid_rules, + report.final_rules, + report.parse_errors.len(), + report.invalid_regexes.len(), + report.duplicate_ids.len(), + report.shadowed_rules.len() + ); + failed |= !report.parse_errors.is_empty() || !report.duplicate_ids.is_empty(); + } + + if check_fixtures { + let rules = load_builtin_rules(); + let report = verify_rule_fixtures(&fixtures_dir, &rules); + println!( + "fixtures: dir={} seen={} passed={} parse_errors={} failures={}", + fixtures_dir.display(), + report.fixtures_seen, + report.passed, + report.parse_errors.len(), + report.failures.len() + ); + failed |= !report.is_clean(); + } + + if failed { + Err("verification failed".to_owned()) + } else { + Ok(()) + } +} + +fn run_reduce(args: &[String]) -> Result<(), String> { + let mut tool_name = "stdin".to_owned(); + let mut command = None; + let mut exit_code = None; + let mut max_inline_chars = None; + let mut path = None; + let mut i = 0; + + while i < args.len() { + match args[i].as_str() { + "--tool-name" => { + i += 1; + tool_name = args + .get(i) + .ok_or_else(|| "--tool-name requires a value".to_owned())? + .clone(); + } + "--command" => { + i += 1; + command = Some( + args.get(i) + .ok_or_else(|| "--command requires a value".to_owned())? + .clone(), + ); + } + "--exit-code" => { + i += 1; + let value = args + .get(i) + .ok_or_else(|| "--exit-code requires a value".to_owned())?; + exit_code = Some( + value + .parse::() + .map_err(|_| "--exit-code must be an integer".to_owned())?, + ); + } + "--max-inline-chars" => { + i += 1; + let value = args + .get(i) + .ok_or_else(|| "--max-inline-chars requires a value".to_owned())?; + max_inline_chars = Some( + value + .parse::() + .map_err(|_| "--max-inline-chars must be a positive integer".to_owned())?, + ); + } + "-h" | "--help" => { + print_reduce_usage(); + return Ok(()); + } + value if value.starts_with('-') && value != "-" => { + return Err(format!("unknown reduce option: {value}")); + } + value => { + if path.replace(value.to_owned()).is_some() { + return Err("reduce accepts at most one path".to_owned()); + } + } + } + i += 1; + } + + let stdout = read_input(path.as_deref())?; + let rules = load_builtin_rules(); + let input = ToolExecutionInput { + tool_name, + command, + exit_code, + stdout: Some(stdout), + ..Default::default() + }; + let options = ReduceOptions { + max_inline_chars, + ..Default::default() + }; + let result = reduce_execution_with_rules(input, &rules, &options); + print!("{}", result.inline_text); + Ok(()) +} + +fn run_reduce_json(args: &[String]) -> Result<(), String> { + let mut pretty = false; + let mut path = None; + let mut i = 0; + + while i < args.len() { + match args[i].as_str() { + "--pretty" => pretty = true, + "-h" | "--help" => { + print_reduce_json_usage(); + return Ok(()); + } + value if value.starts_with('-') && value != "-" => { + return Err(format!("unknown reduce-json option: {value}")); + } + value => { + if path.replace(value.to_owned()).is_some() { + return Err("reduce-json accepts at most one path".to_owned()); + } + } + } + i += 1; + } + + let payload = read_input(path.as_deref())?; + let rules = load_builtin_rules(); + match reduce_json_str(&payload, &rules) { + Ok(response) => { + if pretty { + println!( + "{}", + serde_json::to_string_pretty(&response) + .map_err(|error| format!("failed to serialize response: {error}"))? + ); + } else { + println!( + "{}", + serde_json::to_string(&response) + .map_err(|error| format!("failed to serialize response: {error}"))? + ); + } + Ok(()) + } + Err(error) => { + let json = serde_json::to_string(&error).unwrap_or_else(|_| { + r#"{"code":"internal","message":"serialization failed"}"#.to_owned() + }); + Err(json) + } + } +} + +fn read_input(path: Option<&str>) -> Result { + match path { + Some("-") | None => { + let mut input = String::new(); + io::stdin() + .read_to_string(&mut input) + .map_err(|error| format!("failed to read stdin: {error}"))?; + Ok(input) + } + Some(path) => { + fs::read_to_string(path).map_err(|error| format!("failed to read {path}: {error}")) + } + } +} + +fn print_usage() { + println!("Usage: tinyjuice [options] [path|-]"); + println!(); + println!("Commands:"); + println!(" reduce Reduce plain tool output from a file or stdin"); + println!(" reduce-json Reduce a JSON protocol payload from a file or stdin"); + println!(" verify Verify built-in rules and/or reducer fixtures"); + println!(" discover Report command families still using generic fallback"); + println!(" wrap Run a command and reduce its captured output"); + println!(" ls List tokens from an explicit CCR disk store"); + println!(" cat Print a token from an explicit CCR disk store"); + println!(" stats Report metadata-only CCR disk store stats"); + println!(" doctor Report TinyJuice health checks"); + println!(" install Install TinyJuice integration for a host"); + println!(" uninstall Remove TinyJuice integration from a host"); +} + +fn print_reduce_usage() { + println!( + "Usage: tinyjuice reduce [--tool-name NAME] [--command CMD] [--exit-code N] [--max-inline-chars N] [path|-]" + ); +} + +fn print_reduce_json_usage() { + println!("Usage: tinyjuice reduce-json [--pretty] [path|-]"); +} + +fn print_verify_usage() { + println!("Usage: tinyjuice verify [--rules] [--fixtures [dir]]"); +} + +fn print_discover_usage() { + println!("Usage: tinyjuice discover [--pretty] [path|-]"); + println!("Input is a JSON array or newline-delimited ToolExecutionInput objects."); +} + +fn print_wrap_usage() { + println!( + "Usage: tinyjuice wrap [--tool-name NAME] [--max-inline-chars N] -- command [args...]" + ); +} + +fn print_ls_usage() { + println!("Usage: tinyjuice ls --store-dir DIR"); +} + +fn print_cat_usage() { + println!("Usage: tinyjuice cat --store-dir DIR [--lines START:END | --bytes START:END] TOKEN"); +} + +fn print_stats_usage() { + println!("Usage: tinyjuice stats --store-dir DIR [--pretty]"); +} + +fn print_doctor_usage() { + println!( + "Usage: tinyjuice doctor [host|hooks] [--pretty] [--fixtures [dir]] [--store-dir DIR] [--codex-home DIR] [--openhuman-root DIR]" + ); +} + +fn print_install_usage() { + println!("Usage: tinyjuice install codex [--codex-home DIR] [--local PATH]"); +} + +fn print_uninstall_usage() { + println!("Usage: tinyjuice uninstall codex [--codex-home DIR]"); +} diff --git a/src/ml.rs b/src/ml.rs index 8d1bd7f..707611b 100644 --- a/src/ml.rs +++ b/src/ml.rs @@ -24,6 +24,12 @@ pub fn configure_callback(callback: Option>) { *callback_cell().write().unwrap_or_else(|p| p.into_inner()) = callback; } +#[cfg(test)] +pub(crate) async fn callback_test_guard() -> tokio::sync::MutexGuard<'static, ()> { + static TEST_LOCK: tokio::sync::Mutex<()> = tokio::sync::Mutex::const_new(()); + TEST_LOCK.lock().await +} + /// Compress text through the host callback, if one has been configured. pub async fn compress(text: &str, opts: &CompressOptions) -> Result, String> { let callback = callback_cell() @@ -39,13 +45,10 @@ pub async fn compress(text: &str, opts: &CompressOptions) -> Result = Mutex::const_new(()); #[tokio::test] async fn declines_when_no_callback_is_configured() { - let _guard = TEST_LOCK.lock().await; + let _guard = callback_test_guard().await; configure_callback(None); let result = compress("plain text", &CompressOptions::default()) @@ -57,7 +60,7 @@ mod tests { #[tokio::test] async fn delegates_to_configured_callback_with_owned_options() { - let _guard = TEST_LOCK.lock().await; + let _guard = callback_test_guard().await; configure_callback(Some(Arc::new(|text, opts| { Box::pin(async move { assert_eq!(text, "alpha beta gamma"); @@ -80,7 +83,7 @@ mod tests { #[tokio::test] async fn propagates_callback_errors() { - let _guard = TEST_LOCK.lock().await; + let _guard = callback_test_guard().await; configure_callback(Some(Arc::new(|_, _| { Box::pin(async { Err("runtime offline".to_string()) }) }))); diff --git a/src/observability.rs b/src/observability.rs new file mode 100644 index 0000000..cde3551 --- /dev/null +++ b/src/observability.rs @@ -0,0 +1,264 @@ +//! Host-facing, non-sensitive context usage breakdowns. +//! +//! These structs let an adapter explain why compression triggered without +//! handing TinyJuice raw prompt, tool, memory, or conversation text. + +/// Provider-neutral context bucket categories. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum ContextBucketKind { + SystemPrompt, + ToolDefinitions, + RulesContextFiles, + Skills, + McpTools, + Subagents, + Memory, + Conversation, + Other, +} + +impl ContextBucketKind { + pub fn as_str(self) -> &'static str { + match self { + Self::SystemPrompt => "system_prompt", + Self::ToolDefinitions => "tool_definitions", + Self::RulesContextFiles => "rules_context_files", + Self::Skills => "skills", + Self::McpTools => "mcp_tools", + Self::Subagents => "subagents", + Self::Memory => "memory", + Self::Conversation => "conversation", + Self::Other => "other", + } + } +} + +/// One non-sensitive context usage bucket. +#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ContextBucket { + pub kind: ContextBucketKind, + pub estimated_tokens: usize, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub measured_tokens: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub byte_count: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub item_count: Option, + /// Stable prefix cost that compression should not be blamed for, such as + /// the system prompt and static tool definitions. + pub static_prefix: bool, + /// Whether this bucket is a plausible target for host conversation/content + /// compaction. Static provider costs should usually leave this false. + pub compression_candidate: bool, +} + +impl ContextBucket { + pub fn estimated(kind: ContextBucketKind, estimated_tokens: usize) -> Self { + Self { + kind, + estimated_tokens, + measured_tokens: None, + byte_count: None, + item_count: None, + static_prefix: false, + compression_candidate: false, + } + } + + pub fn system_prompt(estimated_tokens: usize) -> Self { + Self::estimated(ContextBucketKind::SystemPrompt, estimated_tokens).as_static_prefix() + } + + pub fn tool_definitions(estimated_tokens: usize) -> Self { + Self::estimated(ContextBucketKind::ToolDefinitions, estimated_tokens).as_static_prefix() + } + + pub fn conversation(estimated_tokens: usize) -> Self { + Self::estimated(ContextBucketKind::Conversation, estimated_tokens) + .as_compression_candidate() + } + + pub fn memory(estimated_tokens: usize) -> Self { + Self::estimated(ContextBucketKind::Memory, estimated_tokens).as_compression_candidate() + } + + pub fn with_measured_tokens(mut self, measured_tokens: usize) -> Self { + self.measured_tokens = Some(measured_tokens); + self + } + + pub fn with_byte_count(mut self, byte_count: usize) -> Self { + self.byte_count = Some(byte_count); + self + } + + pub fn with_item_count(mut self, item_count: usize) -> Self { + self.item_count = Some(item_count); + self + } + + pub fn as_static_prefix(mut self) -> Self { + self.static_prefix = true; + self.compression_candidate = false; + self + } + + pub fn as_compression_candidate(mut self) -> Self { + self.compression_candidate = true; + self + } + + pub fn effective_tokens(&self) -> usize { + self.measured_tokens.unwrap_or(self.estimated_tokens) + } +} + +/// Non-sensitive context usage report for host UIs and logs. +#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ContextBreakdown { + pub categories: Vec, + pub estimated_total_tokens: usize, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub measured_prompt_tokens: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub context_max_tokens: Option, +} + +impl ContextBreakdown { + pub fn new(categories: Vec) -> Self { + let estimated_total_tokens = categories + .iter() + .map(|bucket| bucket.estimated_tokens) + .sum(); + Self { + categories, + estimated_total_tokens, + measured_prompt_tokens: None, + context_max_tokens: None, + } + } + + pub fn empty() -> Self { + Self::new(Vec::new()) + } + + pub fn add_bucket(&mut self, bucket: ContextBucket) { + self.estimated_total_tokens += bucket.estimated_tokens; + self.categories.push(bucket); + } + + pub fn with_measured_prompt_tokens(mut self, measured_prompt_tokens: usize) -> Self { + self.measured_prompt_tokens = Some(measured_prompt_tokens); + self + } + + pub fn with_context_max_tokens(mut self, context_max_tokens: usize) -> Self { + self.context_max_tokens = Some(context_max_tokens); + self + } + + /// Total prompt tokens to display. Provider-measured usage wins over local + /// estimates when the host can provide it. + pub fn effective_prompt_tokens(&self) -> usize { + self.measured_prompt_tokens.unwrap_or_else(|| { + self.categories + .iter() + .map(ContextBucket::effective_tokens) + .sum() + }) + } + + /// Tokens that belong to stable prompt prefix material rather than + /// compressible conversation/history. + pub fn static_prefix_tokens(&self) -> usize { + self.categories + .iter() + .filter(|bucket| bucket.static_prefix) + .map(ContextBucket::effective_tokens) + .sum() + } + + /// Tokens in buckets the host marked as plausible compaction targets. + pub fn compression_candidate_tokens(&self) -> usize { + self.categories + .iter() + .filter(|bucket| bucket.compression_candidate) + .map(ContextBucket::effective_tokens) + .sum() + } + + pub fn usage_ratio(&self) -> Option { + let max = self.context_max_tokens?; + if max == 0 { + return None; + } + Some(self.effective_prompt_tokens() as f64 / max as f64) + } + + pub fn bucket(&self, kind: ContextBucketKind) -> Option<&ContextBucket> { + self.categories.iter().find(|bucket| bucket.kind == kind) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn breakdown_separates_static_prefix_from_compressible_context() { + let breakdown = ContextBreakdown::new(vec![ + ContextBucket::system_prompt(100), + ContextBucket::tool_definitions(200), + ContextBucket::conversation(700), + ContextBucket::memory(50), + ]) + .with_context_max_tokens(2_000); + + assert_eq!(breakdown.estimated_total_tokens, 1_050); + assert_eq!(breakdown.static_prefix_tokens(), 300); + assert_eq!(breakdown.compression_candidate_tokens(), 750); + assert_eq!(breakdown.effective_prompt_tokens(), 1_050); + assert_eq!(breakdown.usage_ratio(), Some(0.525)); + } + + #[test] + fn measured_prompt_tokens_override_rough_estimate() { + let breakdown = ContextBreakdown::new(vec![ + ContextBucket::system_prompt(100), + ContextBucket::conversation(400), + ]) + .with_measured_prompt_tokens(800); + + assert_eq!(breakdown.estimated_total_tokens, 500); + assert_eq!(breakdown.effective_prompt_tokens(), 800); + } + + #[test] + fn bucket_measured_tokens_override_bucket_estimate() { + let breakdown = ContextBreakdown::new(vec![ + ContextBucket::system_prompt(100), + ContextBucket::conversation(400).with_measured_tokens(250), + ]); + + assert_eq!(breakdown.effective_prompt_tokens(), 350); + assert_eq!(breakdown.compression_candidate_tokens(), 250); + } + + #[test] + fn serialized_breakdown_contains_no_raw_prompt_fields() { + let breakdown = ContextBreakdown::new(vec![ + ContextBucket::estimated(ContextBucketKind::Skills, 42) + .with_item_count(3) + .with_byte_count(1_024), + ]); + let json = serde_json::to_string(&breakdown).unwrap(); + + assert!(json.contains("skills")); + assert!(!json.contains("content")); + assert!(!json.contains("promptText")); + assert!(!json.contains("raw")); + } +} diff --git a/src/pipeline/estimate.rs b/src/pipeline/estimate.rs new file mode 100644 index 0000000..c37c74a --- /dev/null +++ b/src/pipeline/estimate.rs @@ -0,0 +1,580 @@ +use crate::types::ContentKind; + +/// Redacted estimate of how much non-essential bulk a payload appears to carry. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct BloatEstimate { + /// Heuristic score in the range 0..=100. + pub score: u8, + /// Dominant signal behind the score. Never contains raw content. + pub reason: BloatReason, +} + +/// Categorical bloat reason. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum BloatReason { + JsonRows, + JsonSaturation, + DiffContext, + DiffNoise, + LogRepetition, + LogTemplates, + SearchFanout, + SearchClustering, + HtmlMarkup, + CodeBody, + TextRepetition, + LowSignal, +} + +impl BloatReason { + pub fn as_str(self) -> &'static str { + match self { + Self::JsonRows => "json_rows", + Self::JsonSaturation => "json_saturation", + Self::DiffContext => "diff_context", + Self::DiffNoise => "diff_noise", + Self::LogRepetition => "log_repetition", + Self::LogTemplates => "log_templates", + Self::SearchFanout => "search_fanout", + Self::SearchClustering => "search_clustering", + Self::HtmlMarkup => "html_markup", + Self::CodeBody => "code_body", + Self::TextRepetition => "text_repetition", + Self::LowSignal => "low_signal", + } + } +} + +/// Estimate bloat without retaining or reporting raw content. +pub fn estimate_bloat(content: &str, kind: ContentKind) -> BloatEstimate { + match kind { + ContentKind::Json => estimate_json_bloat(content), + ContentKind::Diff => estimate_diff_bloat(content), + ContentKind::Log => estimate_log_bloat(content), + ContentKind::Search => estimate_search_bloat(content), + ContentKind::Html => estimate_html_bloat(content), + ContentKind::Code => estimate_code_bloat(content), + ContentKind::PlainText => estimate_text_bloat(content), + } +} + +fn score_ratio(numerator: usize, denominator: usize) -> u8 { + if denominator == 0 { + return 0; + } + ((numerator.min(denominator) * 100) / denominator) as u8 +} + +fn estimate_json_bloat(content: &str) -> BloatEstimate { + let Ok(value) = serde_json::from_str::(content.trim()) else { + return low(); + }; + let serde_json::Value::Array(rows) = value else { + return low(); + }; + if rows.len() < 2 { + return low(); + } + let object_rows = rows.iter().filter(|row| row.is_object()).count(); + if object_rows == 0 { + return low(); + } + + let mut columns = std::collections::BTreeSet::new(); + let mut row_shapes = std::collections::HashSet::new(); + let mut row_hashes = std::collections::HashSet::new(); + let mut scalar_values_by_key: std::collections::HashMap< + String, + std::collections::HashSet, + > = std::collections::HashMap::new(); + let mut present_cells = 0usize; + + for row in rows.iter().filter_map(|row| row.as_object()) { + let mut shape = Vec::with_capacity(row.len()); + for (key, value) in row { + columns.insert(key.clone()); + shape.push(key.as_str()); + present_cells += 1; + if value.is_string() || value.is_boolean() || value.is_number() || value.is_null() { + scalar_values_by_key + .entry(key.clone()) + .or_default() + .insert(stable_value_hash(value)); + } + } + shape.sort_unstable(); + row_shapes.insert(shape.join("\0")); + row_hashes.insert(stable_object_hash(row)); + } + + let total_cells = object_rows.saturating_mul(columns.len()).max(1); + let sparse_cells = total_cells.saturating_sub(present_cells); + let constant_fields = scalar_values_by_key + .values() + .filter(|values| values.len() == 1) + .count(); + let duplicate_rows = object_rows.saturating_sub(row_hashes.len()); + let saturation_score = (score_ratio(duplicate_rows, object_rows) / 3) + + (score_ratio(constant_fields, columns.len().max(1)) / 3) + + (score_ratio(sparse_cells, total_cells) / 3); + let row_pressure = score_ratio(object_rows.saturating_sub(2), rows.len()) / 2; + let shape_pressure = score_ratio(row_shapes.len().saturating_sub(1), object_rows.max(1)) / 4; + let score = (40 + row_pressure + saturation_score + shape_pressure).min(95); + let reason = if saturation_score >= 20 || duplicate_rows >= object_rows / 4 { + BloatReason::JsonSaturation + } else { + BloatReason::JsonRows + }; + BloatEstimate { score, reason } +} + +fn estimate_diff_bloat(content: &str) -> BloatEstimate { + let mut context_bytes = 0usize; + let mut changed_bytes = 0usize; + let mut noisy_bytes = 0usize; + let mut whitespace_only_bytes = 0usize; + let mut current_file_is_noisy = false; + let mut hunk_lines: Vec<&str> = Vec::new(); + + let flush_hunk = |hunk_lines: &mut Vec<&str>, whitespace_only_bytes: &mut usize| { + if hunk_is_whitespace_only(hunk_lines) { + *whitespace_only_bytes += hunk_lines + .iter() + .filter(|line| is_changed_diff_line(line)) + .map(|line| line.len() + 1) + .sum::(); + } + hunk_lines.clear(); + }; + + for line in content.lines().take(4_000) { + if line.starts_with("diff --git") { + flush_hunk(&mut hunk_lines, &mut whitespace_only_bytes); + current_file_is_noisy = diff_path_is_noisy(line); + continue; + } + if line.starts_with("@@") { + flush_hunk(&mut hunk_lines, &mut whitespace_only_bytes); + continue; + } + if line.starts_with("+++") || line.starts_with("---") { + continue; + } + + if is_changed_diff_line(line) { + let bytes = line.len() + 1; + changed_bytes += bytes; + hunk_lines.push(line); + if current_file_is_noisy { + noisy_bytes += bytes; + } + } else if line.starts_with(' ') { + let bytes = line.len() + 1; + context_bytes += bytes; + hunk_lines.push(line); + if current_file_is_noisy { + noisy_bytes += bytes; + } + } else if !line.trim().is_empty() && current_file_is_noisy { + noisy_bytes += line.len() + 1; + } + } + flush_hunk(&mut hunk_lines, &mut whitespace_only_bytes); + + let total = context_bytes + changed_bytes; + if total == 0 { + return low(); + } + let droppable_noise = (noisy_bytes + whitespace_only_bytes).min(total); + if droppable_noise > 0 { + let score = score_ratio(droppable_noise, total).max(score_ratio(context_bytes, total) / 2); + return BloatEstimate { + score, + reason: BloatReason::DiffNoise, + }; + } + BloatEstimate { + score: score_ratio(context_bytes, total), + reason: BloatReason::DiffContext, + } +} + +fn estimate_log_bloat(content: &str) -> BloatEstimate { + let mut total = 0usize; + let mut repeated = 0usize; + let mut priority = 0usize; + let mut previous = ""; + let mut templates: std::collections::HashMap = std::collections::HashMap::new(); + for line in content.lines().take(4_000) { + let trimmed = line.trim(); + if trimmed.is_empty() { + continue; + } + total += 1; + if log_line_has_priority(trimmed) { + priority += 1; + } + if trimmed == previous { + repeated += 1; + } + previous = trimmed; + let template = log_template(trimmed); + if template != trimmed { + *templates.entry(template).or_default() += 1; + } + } + if total == 0 { + return low(); + } + let template_repetition = templates + .values() + .filter(|count| **count >= 3) + .map(|count| count.saturating_sub(1)) + .sum::(); + let priority_dilution = total.saturating_sub(priority).saturating_sub(80); + let score = score_ratio(repeated + template_repetition + priority_dilution, total).min(95); + if template_repetition >= 10 { + return BloatEstimate { + score, + reason: BloatReason::LogTemplates, + }; + } + BloatEstimate { + score, + reason: BloatReason::LogRepetition, + } +} + +fn estimate_search_bloat(content: &str) -> BloatEstimate { + let mut total = 0usize; + let mut paths: std::collections::HashMap<&str, usize> = std::collections::HashMap::new(); + for line in content.lines().take(2_000) { + let Some((path, _rest)) = line.split_once(':') else { + continue; + }; + if path.trim().is_empty() { + continue; + } + total += 1; + *paths.entry(path).or_default() += 1; + } + if total == 0 { + return low(); + } + let fanout = paths.len(); + let max_cluster = paths.values().copied().max().unwrap_or(0); + if total >= 20 && score_ratio(max_cluster, total) >= 60 { + return BloatEstimate { + score: score_ratio(max_cluster, total).min(95), + reason: BloatReason::SearchClustering, + }; + } + BloatEstimate { + score: (score_ratio(total.saturating_sub(20), total) / 2 + + score_ratio(fanout.saturating_sub(3), fanout.max(1)) / 2) + .min(95), + reason: BloatReason::SearchFanout, + } +} + +fn estimate_html_bloat(content: &str) -> BloatEstimate { + let tag_bytes = content + .bytes() + .filter(|byte| matches!(byte, b'<' | b'>')) + .count(); + BloatEstimate { + score: score_ratio(tag_bytes * 10, content.len().max(1)).min(95), + reason: BloatReason::HtmlMarkup, + } +} + +fn estimate_code_bloat(content: &str) -> BloatEstimate { + let mut total = 0usize; + let mut body_like = 0usize; + for line in content.lines().take(2_000) { + let trimmed = line.trim(); + if trimmed.is_empty() { + continue; + } + total += 1; + if trimmed == "{" || trimmed == "}" || trimmed.ends_with(';') || trimmed.contains(" = ") { + body_like += 1; + } + } + if total == 0 { + return low(); + } + BloatEstimate { + score: score_ratio(body_like, total), + reason: BloatReason::CodeBody, + } +} + +fn estimate_text_bloat(content: &str) -> BloatEstimate { + let mut total = 0usize; + let mut repeated = 0usize; + let mut seen = std::collections::HashSet::new(); + for line in content.lines().take(2_000) { + let trimmed = line.trim(); + if trimmed.is_empty() { + continue; + } + total += 1; + if !seen.insert(trimmed) { + repeated += 1; + } + } + if total == 0 || repeated == 0 { + return low(); + } + BloatEstimate { + score: score_ratio(repeated, total), + reason: BloatReason::TextRepetition, + } +} + +fn low() -> BloatEstimate { + BloatEstimate { + score: 0, + reason: BloatReason::LowSignal, + } +} + +fn stable_value_hash(value: &serde_json::Value) -> u64 { + stable_hash(serde_json::to_string(value).unwrap_or_default().as_bytes()) +} + +fn stable_object_hash(object: &serde_json::Map) -> u64 { + let mut bytes = Vec::new(); + for (key, value) in object { + bytes.extend_from_slice(key.as_bytes()); + bytes.push(0); + bytes.extend_from_slice(serde_json::to_string(value).unwrap_or_default().as_bytes()); + bytes.push(0xff); + } + stable_hash(&bytes) +} + +fn stable_hash(bytes: &[u8]) -> u64 { + let mut hash = 0xcbf29ce484222325u64; + for byte in bytes { + hash ^= u64::from(*byte); + hash = hash.wrapping_mul(0x100000001b3); + } + hash +} + +fn is_changed_diff_line(line: &str) -> bool { + (line.starts_with('+') && !line.starts_with("+++")) + || (line.starts_with('-') && !line.starts_with("---")) +} + +fn hunk_is_whitespace_only(lines: &[&str]) -> bool { + let removed: Vec = lines + .iter() + .filter_map(|line| line.strip_prefix('-')) + .map(strip_ascii_whitespace) + .collect(); + let added: Vec = lines + .iter() + .filter_map(|line| line.strip_prefix('+')) + .map(strip_ascii_whitespace) + .collect(); + !removed.is_empty() && removed.len() == added.len() && removed == added +} + +fn strip_ascii_whitespace(s: &str) -> String { + s.chars().filter(|c| !c.is_ascii_whitespace()).collect() +} + +fn diff_path_is_noisy(diff_git_line: &str) -> bool { + let line = diff_git_line.to_ascii_lowercase(); + const LOCKFILE_PATTERNS: &[&str] = &[ + "cargo.lock", + "package-lock.json", + "pnpm-lock.yaml", + "yarn.lock", + "composer.lock", + "poetry.lock", + "gemfile.lock", + "go.sum", + ]; + const GENERATED_BUNDLE_PATTERNS: &[&str] = &[".min.js", ".min.css", ".map"]; + LOCKFILE_PATTERNS + .iter() + .chain(GENERATED_BUNDLE_PATTERNS) + .any(|pattern| line.contains(pattern)) +} + +fn log_line_has_priority(line: &str) -> bool { + const PRIORITY_MARKERS: &[&str] = &[ + "error", + "failed", + "failure", + "panic", + "exception", + "warning", + "warn", + "fatal", + "traceback", + ]; + let lower = line.to_ascii_lowercase(); + PRIORITY_MARKERS.iter().any(|marker| lower.contains(marker)) +} + +fn log_template(line: &str) -> String { + let mut out = String::with_capacity(line.len()); + let mut chars = line.chars().peekable(); + while let Some(ch) = chars.next() { + if ch == '0' && chars.peek() == Some(&'x') { + out.push_str("0x#"); + chars.next(); + while chars.peek().is_some_and(|next| next.is_ascii_hexdigit()) { + chars.next(); + } + } else if ch.is_ascii_digit() { + out.push('#'); + while chars.peek().is_some_and(|next| next.is_ascii_digit()) { + chars.next(); + } + } else { + out.push(ch); + } + } + out +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn json_rows_estimate_is_redacted() { + let content = r#"[{"secret":"a"},{"secret":"b"},{"secret":"c"}]"#; + let estimate = estimate_bloat(content, ContentKind::Json); + assert!(matches!( + estimate.reason, + BloatReason::JsonRows | BloatReason::JsonSaturation + )); + assert!(estimate.score > 0); + assert!(!estimate.reason.as_str().contains("secret")); + } + + #[test] + fn json_saturation_detects_duplicate_constant_rows() { + let mut rows = Vec::new(); + for i in 0..80 { + rows.push(format!( + r#"{{"id":{},"status":"ok","region":"us","shape":"same"}}"#, + i % 4 + )); + } + let content = format!("[{}]", rows.join(",")); + let estimate = estimate_bloat(&content, ContentKind::Json); + + assert_eq!(estimate.reason, BloatReason::JsonSaturation); + assert!(estimate.score >= 60, "{estimate:?}"); + } + + #[test] + fn diff_context_estimate_detects_context_heavy_payload() { + let mut diff = String::from("diff --git a/a b/a\n@@ -1,8 +1,8 @@\n"); + for i in 0..80 { + diff.push_str(&format!(" context line {i}\n")); + } + diff.push_str("-old\n+new\n"); + let estimate = estimate_bloat(&diff, ContentKind::Diff); + assert_eq!(estimate.reason, BloatReason::DiffContext); + assert!(estimate.score > 80); + } + + #[test] + fn diff_noise_estimate_detects_lockfile_hunks() { + let mut diff = String::from("diff --git a/Cargo.lock b/Cargo.lock\n"); + diff.push_str("@@ -1,20 +1,40 @@\n"); + for i in 0..40 { + diff.push_str(&format!("+ dep version {i}\n")); + } + for i in 0..20 { + diff.push_str(&format!("- old dep version {i}\n")); + } + + let estimate = estimate_bloat(&diff, ContentKind::Diff); + + assert_eq!(estimate.reason, BloatReason::DiffNoise); + assert!(estimate.score > 80, "{estimate:?}"); + } + + #[test] + fn diff_noise_estimate_detects_generated_bundle_hunks() { + let mut diff = String::from("diff --git a/dist/app.min.js b/dist/app.min.js\n"); + diff.push_str("@@ -1,40 +1,40 @@\n"); + for i in 0..40 { + diff.push_str(&format!("+ minified chunk {i}\n")); + } + for i in 0..40 { + diff.push_str(&format!("- previous minified chunk {i}\n")); + } + + let estimate = estimate_bloat(&diff, ContentKind::Diff); + + assert_eq!(estimate.reason, BloatReason::DiffNoise); + assert!(estimate.score > 80, "{estimate:?}"); + } + + #[test] + fn diff_noise_estimate_detects_whitespace_only_hunks() { + let diff = "\ +diff --git a/src/lib.rs b/src/lib.rs +@@ -1,2 +1,2 @@ +-fn main(){println!(\"hi\");} ++fn main() { println!(\"hi\"); } + context +"; + + let estimate = estimate_bloat(diff, ContentKind::Diff); + + assert_eq!(estimate.reason, BloatReason::DiffNoise); + assert!(estimate.score > 0, "{estimate:?}"); + } + + #[test] + fn log_template_estimate_detects_repeated_variants() { + let mut log = String::new(); + for i in 0..80 { + log.push_str(&format!( + "2026-07-05T12:{:02}:00Z worker-{i} processed item id={}\n", + i % 60, + 10_000 + i + )); + } + + let estimate = estimate_bloat(&log, ContentKind::Log); + + assert_eq!(estimate.reason, BloatReason::LogTemplates); + assert!(estimate.score > 50, "{estimate:?}"); + } + + #[test] + fn search_clustering_estimate_detects_single_file_fan_in() { + let mut output = String::new(); + for i in 0..40 { + output.push_str(&format!("src/lib.rs:{i}:needle match {i}\n")); + } + for i in 0..5 { + output.push_str(&format!("src/other.rs:{i}:needle match {i}\n")); + } + + let estimate = estimate_bloat(&output, ContentKind::Search); + + assert_eq!(estimate.reason, BloatReason::SearchClustering); + assert!(estimate.score >= 80, "{estimate:?}"); + } + + #[test] + fn plain_unique_text_is_low_signal() { + let estimate = estimate_bloat("alpha\nbeta\ngamma", ContentKind::PlainText); + assert_eq!(estimate.reason, BloatReason::LowSignal); + assert_eq!(estimate.score, 0); + } +} diff --git a/src/pipeline/mod.rs b/src/pipeline/mod.rs new file mode 100644 index 0000000..068f077 --- /dev/null +++ b/src/pipeline/mod.rs @@ -0,0 +1,18 @@ +//! Typed compression pipeline primitives. +//! +//! These types make TinyJuice's safety contract explicit for new code: +//! lossless reformats produce ordinary transform output, while lossy offloads +//! must carry a CCR token from a store put result before model-facing text can +//! be emitted. + +mod estimate; +mod report; +mod runner; +mod transform; + +pub use estimate::{BloatEstimate, BloatReason, estimate_bloat}; +pub use report::{PipelineReport, PipelineSkipReason, PipelineStep}; +pub use runner::{TypedPipelineOutput, run_typed_pipeline}; +pub use transform::{ + OffloadOutput, OffloadTransform, PipelineInput, ReformatTransform, TransformOutput, +}; diff --git a/src/pipeline/report.rs b/src/pipeline/report.rs new file mode 100644 index 0000000..b7633c2 --- /dev/null +++ b/src/pipeline/report.rs @@ -0,0 +1,104 @@ +use crate::pipeline::BloatEstimate; +use crate::types::{CompressorKind, ContentKind}; + +/// A transform that ran or was considered by the pipeline. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct PipelineStep { + pub name: &'static str, + pub compressor: Option, +} + +/// Redacted reason the pipeline declined to transform content. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum PipelineSkipReason { + RouterDisabled, + BelowMinBytes, + NoCompressor, + NoSavings, + LowBloat, + CcrDisabled, + CcrNotRetained, + FooterWouldGrow, + RecoveryTool, + Other(&'static str), +} + +impl PipelineSkipReason { + pub fn as_str(&self) -> &'static str { + match self { + Self::RouterDisabled => "router_disabled", + Self::BelowMinBytes => "below_min_bytes", + Self::NoCompressor => "no_compressor", + Self::NoSavings => "no_savings", + Self::LowBloat => "low_bloat", + Self::CcrDisabled => "ccr_disabled", + Self::CcrNotRetained => "ccr_not_retained", + Self::FooterWouldGrow => "footer_would_grow", + Self::RecoveryTool => "recovery_tool", + Self::Other(reason) => reason, + } + } +} + +/// Non-sensitive report for pipeline decisions. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct PipelineReport { + pub content_kind: ContentKind, + pub original_bytes: usize, + pub compacted_bytes: usize, + pub bloat_estimate: Option, + pub applied_steps: Vec, + pub skipped_steps: Vec, + pub ccr_tokens: Vec, + pub lossy: bool, + pub skip_reason: Option, +} + +impl PipelineReport { + pub fn applied( + content_kind: ContentKind, + original_bytes: usize, + compacted_bytes: usize, + compressor: CompressorKind, + lossy: bool, + ccr_token: Option, + ) -> Self { + Self { + content_kind, + original_bytes, + compacted_bytes, + bloat_estimate: None, + applied_steps: vec![PipelineStep { + name: "compat_router", + compressor: Some(compressor), + }], + skipped_steps: Vec::new(), + ccr_tokens: ccr_token.into_iter().collect(), + lossy, + skip_reason: None, + } + } + + pub fn passthrough( + content_kind: ContentKind, + original_bytes: usize, + skip_reason: PipelineSkipReason, + ) -> Self { + Self { + content_kind, + original_bytes, + compacted_bytes: original_bytes, + bloat_estimate: None, + applied_steps: Vec::new(), + skipped_steps: Vec::new(), + ccr_tokens: Vec::new(), + lossy: false, + skip_reason: Some(skip_reason), + } + } + + pub fn with_bloat_estimate(mut self, bloat_estimate: BloatEstimate) -> Self { + self.bloat_estimate = Some(bloat_estimate); + self + } +} diff --git a/src/pipeline/runner.rs b/src/pipeline/runner.rs new file mode 100644 index 0000000..86bd7d7 --- /dev/null +++ b/src/pipeline/runner.rs @@ -0,0 +1,278 @@ +use crate::cache::CcrStore; +use crate::pipeline::{ + OffloadTransform, PipelineInput, PipelineReport, PipelineSkipReason, PipelineStep, + ReformatTransform, +}; +use crate::types::CompressorKind; + +/// Output from a typed pipeline run. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct TypedPipelineOutput { + pub text: String, + pub kind: CompressorKind, + pub lossy: bool, + pub ccr_token: Option, + pub report: PipelineReport, +} + +/// Run a small typed transform pipeline. +/// +/// Reformat transforms run first and may update the current body without CCR. +/// Offload transforms run after reformats, but the [`PipelineInput`] still +/// carries `original_content`, so lossy transforms can retain the true original +/// rather than an intermediate reformat. +pub fn run_typed_pipeline( + input: PipelineInput<'_>, + reformat_transforms: &[&dyn ReformatTransform], + offload_transforms: &[&dyn OffloadTransform], + store: &dyn CcrStore, +) -> TypedPipelineOutput { + let mut current = input.content.to_string(); + let mut kind = CompressorKind::None; + let mut applied_steps = Vec::new(); + let mut skipped_steps = Vec::new(); + let mut offload_declined = false; + + for transform in reformat_transforms { + let step = PipelineStep { + name: transform.name(), + compressor: None, + }; + let current_input = PipelineInput { + content: ¤t, + original_content: input.original_content, + content_kind: input.content_kind, + original_bytes: input.original_bytes, + }; + if !transform.applies_to(¤t_input) { + skipped_steps.push(step); + continue; + } + let Some(output) = transform.apply(¤t_input) else { + skipped_steps.push(step); + continue; + }; + if output.text.len() >= current.len() { + skipped_steps.push(PipelineStep { + name: transform.name(), + compressor: Some(output.kind), + }); + continue; + } + kind = output.kind; + current = output.text; + applied_steps.push(PipelineStep { + name: transform.name(), + compressor: Some(output.kind), + }); + } + + for transform in offload_transforms { + let current_input = PipelineInput { + content: ¤t, + original_content: input.original_content, + content_kind: input.content_kind, + original_bytes: input.original_bytes, + }; + if transform.estimate_bloat(¤t_input) <= 0.0 { + skipped_steps.push(PipelineStep { + name: transform.name(), + compressor: None, + }); + continue; + } + let Some(output) = transform.apply(¤t_input, store) else { + offload_declined = true; + skipped_steps.push(PipelineStep { + name: transform.name(), + compressor: None, + }); + continue; + }; + let text = output.text().to_string(); + let token = output.token().to_string(); + kind = output.kind(); + applied_steps.push(PipelineStep { + name: transform.name(), + compressor: Some(kind), + }); + let report = PipelineReport { + content_kind: input.content_kind, + original_bytes: input.original_bytes, + compacted_bytes: text.len(), + bloat_estimate: None, + applied_steps, + skipped_steps, + ccr_tokens: vec![token.clone()], + lossy: true, + skip_reason: None, + }; + return TypedPipelineOutput { + text, + kind, + lossy: true, + ccr_token: Some(token), + report, + }; + } + + let skip_reason = if applied_steps.is_empty() { + Some(if offload_declined { + PipelineSkipReason::CcrNotRetained + } else { + PipelineSkipReason::NoCompressor + }) + } else { + None + }; + let report = PipelineReport { + content_kind: input.content_kind, + original_bytes: input.original_bytes, + compacted_bytes: current.len(), + bloat_estimate: None, + applied_steps, + skipped_steps, + ccr_tokens: Vec::new(), + lossy: false, + skip_reason, + }; + TypedPipelineOutput { + text: current, + kind, + lossy: false, + ccr_token: None, + report, + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::cache::MemoryCcrStore; + use crate::pipeline::{OffloadOutput, TransformOutput}; + use crate::types::ContentKind; + + struct SpaceReformat; + + impl ReformatTransform for SpaceReformat { + fn name(&self) -> &'static str { + "space_reformat" + } + + fn applies_to(&self, input: &PipelineInput<'_>) -> bool { + input.content_kind == ContentKind::PlainText && input.content.contains(" ") + } + + fn apply(&self, input: &PipelineInput<'_>) -> Option { + Some(TransformOutput::new( + input + .content + .split_whitespace() + .collect::>() + .join(" "), + CompressorKind::Generic, + )) + } + } + + struct PrefixOffload; + + impl OffloadTransform for PrefixOffload { + fn name(&self) -> &'static str { + "prefix_offload" + } + + fn estimate_bloat(&self, input: &PipelineInput<'_>) -> f32 { + if input.content.len() > 12 { 1.0 } else { 0.0 } + } + + fn apply(&self, input: &PipelineInput<'_>, store: &dyn CcrStore) -> Option { + let body = input.content.chars().take(8).collect::(); + OffloadOutput::from_retained_put( + body, + CompressorKind::Generic, + store.put(input.original_content), + ) + } + } + + fn input(content: &str) -> PipelineInput<'_> { + PipelineInput { + content, + original_content: content, + content_kind: ContentKind::PlainText, + original_bytes: content.len(), + } + } + + #[test] + fn no_transforms_returns_passthrough_report() { + let store = MemoryCcrStore::new(4, 1024); + let out = run_typed_pipeline(input("plain"), &[], &[], &store); + + assert_eq!(out.text, "plain"); + assert!(!out.lossy); + assert_eq!( + out.report.skip_reason, + Some(PipelineSkipReason::NoCompressor) + ); + assert!(out.report.applied_steps.is_empty()); + } + + #[test] + fn reformat_transform_runs_without_ccr() { + let store = MemoryCcrStore::new(4, 1024); + let reformat: [&dyn ReformatTransform; 1] = [&SpaceReformat]; + let out = run_typed_pipeline(input("alpha beta gamma"), &reformat, &[], &store); + + assert_eq!(out.text, "alpha beta gamma"); + assert!(!out.lossy); + assert_eq!(out.kind, CompressorKind::Generic); + assert_eq!(out.report.applied_steps[0].name, "space_reformat"); + assert!(out.report.ccr_tokens.is_empty()); + } + + #[test] + fn offload_transform_requires_retained_original() { + let store = MemoryCcrStore::new(4, 1024); + let offload: [&dyn OffloadTransform; 1] = [&PrefixOffload]; + let original = "alpha beta gamma delta"; + let out = run_typed_pipeline(input(original), &[], &offload, &store); + + let token = out.ccr_token.expect("offload retained original"); + assert_eq!(out.text, "alpha be"); + assert!(out.lossy); + assert_eq!(store.get(&token).as_deref(), Some(original)); + assert_eq!(out.report.ccr_tokens, vec![token]); + } + + #[test] + fn mixed_reformat_and_offload_retains_true_original() { + let store = MemoryCcrStore::new(4, 1024); + let reformat: [&dyn ReformatTransform; 1] = [&SpaceReformat]; + let offload: [&dyn OffloadTransform; 1] = [&PrefixOffload]; + let original = "alpha beta gamma delta"; + let out = run_typed_pipeline(input(original), &reformat, &offload, &store); + + let token = out.ccr_token.expect("offload retained original"); + assert_eq!(out.text, "alpha be"); + assert_eq!(out.report.applied_steps.len(), 2); + assert_eq!(store.get(&token).as_deref(), Some(original)); + } + + #[test] + fn offload_store_failure_declines_lossy_output() { + let store = MemoryCcrStore::new(1, 4); + let offload: [&dyn OffloadTransform; 1] = [&PrefixOffload]; + let original = "alpha beta gamma delta"; + let out = run_typed_pipeline(input(original), &[], &offload, &store); + + assert_eq!(out.text, original); + assert!(!out.lossy); + assert_eq!(out.ccr_token, None); + assert_eq!( + out.report.skip_reason, + Some(PipelineSkipReason::CcrNotRetained) + ); + } +} diff --git a/src/pipeline/transform.rs b/src/pipeline/transform.rs new file mode 100644 index 0000000..348a942 --- /dev/null +++ b/src/pipeline/transform.rs @@ -0,0 +1,123 @@ +use crate::cache::{CcrPutResult, CcrStore}; +use crate::types::{CompressInput, CompressorKind, ContentKind}; +use std::collections::HashMap; + +/// Input passed to typed transforms. +#[derive(Debug, Clone, Copy)] +pub struct PipelineInput<'a> { + pub content: &'a str, + pub original_content: &'a str, + pub content_kind: ContentKind, + pub original_bytes: usize, +} + +impl<'a> From<&CompressInput<'a>> for PipelineInput<'a> { + fn from(input: &CompressInput<'a>) -> Self { + Self { + content: input.content, + original_content: input.content, + content_kind: input.kind, + original_bytes: input.original_bytes, + } + } +} + +/// Lossless transform output. +#[derive(Debug, Clone)] +pub struct TransformOutput { + pub text: String, + pub kind: CompressorKind, + pub facts: Option>, +} + +impl TransformOutput { + pub fn new(text: String, kind: CompressorKind) -> Self { + Self { + text, + kind, + facts: None, + } + } +} + +/// Lossy transform output with a verified retained CCR token. +#[derive(Debug, Clone)] +pub struct OffloadOutput { + text: String, + kind: CompressorKind, + token: String, + facts: Option>, +} + +impl OffloadOutput { + /// Construct an offload output only from a retained CCR put result. + pub fn from_retained_put( + text: String, + kind: CompressorKind, + put: CcrPutResult, + ) -> Option { + put.retained().then(|| Self { + text, + kind, + token: put.token().to_string(), + facts: None, + }) + } + + pub fn text(&self) -> &str { + &self.text + } + + pub fn kind(&self) -> CompressorKind { + self.kind + } + + pub fn token(&self) -> &str { + &self.token + } + + pub fn facts(&self) -> Option<&HashMap> { + self.facts.as_ref() + } +} + +pub trait ReformatTransform { + fn name(&self) -> &'static str; + fn applies_to(&self, input: &PipelineInput<'_>) -> bool; + fn apply(&self, input: &PipelineInput<'_>) -> Option; +} + +pub trait OffloadTransform { + fn name(&self) -> &'static str; + fn estimate_bloat(&self, input: &PipelineInput<'_>) -> f32; + fn apply(&self, input: &PipelineInput<'_>, store: &dyn CcrStore) -> Option; +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn offload_output_requires_retained_ccr_put() { + let token = "a1b2c3d4".repeat(4); + let rejected = CcrPutResult::new(token.clone(), false); + assert!( + OffloadOutput::from_retained_put( + "partial".to_string(), + CompressorKind::Generic, + rejected, + ) + .is_none() + ); + + let retained = CcrPutResult::new(token.clone(), true); + let out = OffloadOutput::from_retained_put( + "partial".to_string(), + CompressorKind::Generic, + retained, + ) + .expect("retained CCR put constructs offload output"); + assert_eq!(out.text(), "partial"); + assert_eq!(out.token(), token); + } +} diff --git a/src/policy/mod.rs b/src/policy/mod.rs new file mode 100644 index 0000000..6804d75 --- /dev/null +++ b/src/policy/mod.rs @@ -0,0 +1,8 @@ +//! Host-facing compaction policy helpers. + +pub mod shell; + +pub use shell::{ + ShellCompactionPolicy, ShellPolicyDecision, apply_shell_compaction_policy, + is_file_content_inspection_command, +}; diff --git a/src/policy/shell.rs b/src/policy/shell.rs new file mode 100644 index 0000000..106a101 --- /dev/null +++ b/src/policy/shell.rs @@ -0,0 +1,444 @@ +use std::path::Path; + +use crate::types::ToolExecutionInput; + +/// Host policy for shell-command output. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ShellCompactionPolicy { + CompactAll, + SkipAll, + SkipFileContent, + AllowSafeInventory, +} + +/// Redacted decision for shell-command output. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ShellPolicyDecision { + Compact, + SkipAll, + SkipFileContent, + SkipMixedShell, + SkipUnsafeAction, +} + +impl ShellPolicyDecision { + pub fn as_str(self) -> &'static str { + match self { + Self::Compact => "compact", + Self::SkipAll => "skip_all", + Self::SkipFileContent => "skip_file_content", + Self::SkipMixedShell => "skip_mixed_shell", + Self::SkipUnsafeAction => "skip_unsafe_action", + } + } + + pub fn should_compact(self) -> bool { + self == Self::Compact + } +} + +/// Apply shell-output compaction policy to a command-bearing tool input. +pub fn apply_shell_compaction_policy( + input: &ToolExecutionInput, + policy: ShellCompactionPolicy, +) -> ShellPolicyDecision { + match policy { + ShellCompactionPolicy::CompactAll => ShellPolicyDecision::Compact, + ShellCompactionPolicy::SkipAll => ShellPolicyDecision::SkipAll, + ShellCompactionPolicy::SkipFileContent => { + if command_parts(input).is_some_and(|parts| parts.contains_file_content_read()) { + ShellPolicyDecision::SkipFileContent + } else { + ShellPolicyDecision::Compact + } + } + ShellCompactionPolicy::AllowSafeInventory => { + let Some(parts) = command_parts(input) else { + return ShellPolicyDecision::Compact; + }; + if parts.has_mixed_sequence { + return ShellPolicyDecision::SkipMixedShell; + } + if parts.has_unsafe_action() { + return ShellPolicyDecision::SkipUnsafeAction; + } + if parts.contains_file_content_read() { + return ShellPolicyDecision::SkipFileContent; + } + ShellPolicyDecision::Compact + } + } +} + +/// True when the command is a well-known file-content inspection tool. +pub fn is_file_content_inspection_command(input: &ToolExecutionInput) -> bool { + command_parts(input).is_some_and(|parts| parts.contains_file_content_read()) +} + +fn command_parts(input: &ToolExecutionInput) -> Option { + if let Some(command) = input.command.as_deref().filter(|c| !c.trim().is_empty()) { + return parse_command(command); + } + + let argv = input.argv.as_deref()?.to_vec(); + if argv.is_empty() { + return None; + } + Some(CommandParts { + has_redirect: argv.iter().any(|token| is_redirect_token(token)), + segments: vec![argv], + has_mixed_sequence: false, + }) +} + +struct CommandParts { + segments: Vec>, + has_mixed_sequence: bool, + has_redirect: bool, +} + +impl CommandParts { + fn contains_file_content_read(&self) -> bool { + self.segments + .first() + .is_some_and(|tokens| is_file_content_read(tokens)) + } + + fn has_unsafe_action(&self) -> bool { + self.has_redirect + || self + .segments + .iter() + .any(|tokens| contains_unsafe_action(tokens)) + } +} + +fn parse_command(command: &str) -> Option { + let stripped = strip_leading_cd_prefix(command.trim()); + if stripped.is_empty() { + return None; + } + + let has_mixed_sequence = contains_sequence_operator(stripped); + let has_redirect = contains_unquoted_redirect(stripped); + let segments: Vec> = split_unquoted_pipes(stripped) + .into_iter() + .map(crate::reduce::tokenize_command) + .filter(|tokens| !tokens.is_empty()) + .collect(); + + if segments.is_empty() { + None + } else { + Some(CommandParts { + segments, + has_mixed_sequence, + has_redirect, + }) + } +} + +fn strip_leading_cd_prefix(mut command: &str) -> &str { + loop { + let Some((left, right)) = split_once_unquoted_and(command) else { + return command.trim(); + }; + let tokens = crate::reduce::tokenize_command(left); + if tokens.first().map(|s| s.as_str()) != Some("cd") || tokens.len() < 2 { + return command.trim(); + } + command = right.trim(); + } +} + +fn split_once_unquoted_and(s: &str) -> Option<(&str, &str)> { + let bytes = s.as_bytes(); + let mut quote: Option = None; + let mut escaping = false; + let mut i = 0; + while i + 1 < bytes.len() { + let ch = s[i..].chars().next()?; + if escaping { + escaping = false; + i += ch.len_utf8(); + continue; + } + if ch == '\\' { + escaping = true; + i += ch.len_utf8(); + continue; + } + if let Some(q) = quote { + if ch == q { + quote = None; + } + i += ch.len_utf8(); + continue; + } + if ch == '\'' || ch == '"' { + quote = Some(ch); + i += ch.len_utf8(); + continue; + } + if bytes[i] == b'&' && bytes[i + 1] == b'&' { + return Some((&s[..i], &s[i + 2..])); + } + i += ch.len_utf8(); + } + None +} + +fn contains_sequence_operator(s: &str) -> bool { + let bytes = s.as_bytes(); + let mut quote: Option = None; + let mut escaping = false; + let mut i = 0; + while i < bytes.len() { + let ch = s[i..].chars().next().expect("valid char boundary"); + if escaping { + escaping = false; + i += ch.len_utf8(); + continue; + } + if ch == '\\' { + escaping = true; + i += ch.len_utf8(); + continue; + } + if let Some(q) = quote { + if ch == q { + quote = None; + } + i += ch.len_utf8(); + continue; + } + if ch == '\'' || ch == '"' { + quote = Some(ch); + i += ch.len_utf8(); + continue; + } + if ch == ';' || ch == '\n' { + return true; + } + if i + 1 < bytes.len() + && ((bytes[i] == b'&' && bytes[i + 1] == b'&') + || (bytes[i] == b'|' && bytes[i + 1] == b'|')) + { + return true; + } + i += ch.len_utf8(); + } + false +} + +fn contains_unquoted_redirect(s: &str) -> bool { + let mut quote: Option = None; + let mut escaping = false; + for ch in s.chars() { + if escaping { + escaping = false; + continue; + } + if ch == '\\' { + escaping = true; + continue; + } + if let Some(q) = quote { + if ch == q { + quote = None; + } + continue; + } + if ch == '\'' || ch == '"' { + quote = Some(ch); + continue; + } + if ch == '<' || ch == '>' { + return true; + } + } + false +} + +fn split_unquoted_pipes(s: &str) -> Vec<&str> { + let bytes = s.as_bytes(); + let mut out = Vec::new(); + let mut quote: Option = None; + let mut escaping = false; + let mut start = 0; + let mut i = 0; + while i < bytes.len() { + let ch = s[i..].chars().next().expect("valid char boundary"); + if escaping { + escaping = false; + i += ch.len_utf8(); + continue; + } + if ch == '\\' { + escaping = true; + i += ch.len_utf8(); + continue; + } + if let Some(q) = quote { + if ch == q { + quote = None; + } + i += ch.len_utf8(); + continue; + } + if ch == '\'' || ch == '"' { + quote = Some(ch); + i += ch.len_utf8(); + continue; + } + if ch == '|' { + if i + 1 < bytes.len() && bytes[i + 1] == b'|' { + i += 2; + continue; + } + out.push(s[start..i].trim()); + start = i + 1; + } + i += ch.len_utf8(); + } + out.push(s[start..].trim()); + out +} + +fn argv0(tokens: &[String]) -> Option { + tokens.first().map(|cmd| { + Path::new(cmd) + .file_name() + .map(|name| name.to_string_lossy().to_string()) + .unwrap_or_else(|| cmd.to_string()) + }) +} + +fn is_file_content_read(tokens: &[String]) -> bool { + let Some(cmd) = argv0(tokens) else { + return false; + }; + matches!( + cmd.as_str(), + "cat" | "nl" | "bat" | "batcat" | "jq" | "yq" | "head" | "tail" | "sed" + ) +} + +fn contains_unsafe_action(tokens: &[String]) -> bool { + let Some(cmd) = argv0(tokens) else { + return false; + }; + if tokens.iter().any(|token| is_redirect_token(token)) { + return true; + } + match cmd.as_str() { + "find" => tokens.iter().skip(1).any(|t| { + matches!( + t.as_str(), + "-exec" | "-execdir" | "-ok" | "-okdir" | "-delete" + ) + }), + "fd" => tokens.iter().skip(1).any(|t| { + matches!( + t.as_str(), + "-x" | "-X" | "--exec" | "--exec-batch" | "--batch-size" + ) + }), + "rm" | "mv" | "cp" | "install" | "chmod" | "chown" | "ln" | "mkdir" | "rmdir" => true, + "git" => tokens.get(1).is_some_and(|sub| { + matches!( + sub.as_str(), + "checkout" + | "switch" + | "reset" + | "clean" + | "restore" + | "apply" + | "am" + | "merge" + | "rebase" + | "commit" + ) + }), + _ => false, + } +} + +fn is_redirect_token(token: &str) -> bool { + token.contains('>') || token.contains('<') +} + +#[cfg(test)] +mod tests { + use super::*; + + fn input(command: &str) -> ToolExecutionInput { + ToolExecutionInput { + tool_name: "shell".to_owned(), + command: Some(command.to_owned()), + ..Default::default() + } + } + + fn decide(command: &str) -> ShellPolicyDecision { + apply_shell_compaction_policy(&input(command), ShellCompactionPolicy::AllowSafeInventory) + } + + #[test] + fn exact_file_reads_are_skipped() { + assert_eq!( + decide("cat src/lib.rs"), + ShellPolicyDecision::SkipFileContent + ); + assert_eq!( + decide("sed -n '1,200p' src/lib.rs"), + ShellPolicyDecision::SkipFileContent + ); + assert_eq!( + decide("jq . package.json"), + ShellPolicyDecision::SkipFileContent + ); + } + + #[test] + fn inventory_pipelines_are_allowed() { + assert_eq!( + decide("find . -type f | sort | head -n 20"), + ShellPolicyDecision::Compact + ); + assert_eq!(decide("rg --files"), ShellPolicyDecision::Compact); + assert_eq!(decide("git ls-files"), ShellPolicyDecision::Compact); + assert_eq!( + decide("cd crate && rg --files"), + ShellPolicyDecision::Compact + ); + } + + #[test] + fn unsafe_inventory_actions_are_skipped() { + assert_eq!( + decide(r"find . -exec cat {} \;"), + ShellPolicyDecision::SkipUnsafeAction + ); + assert_eq!( + decide("fd --exec cat {}"), + ShellPolicyDecision::SkipUnsafeAction + ); + assert_eq!( + decide("find . -type f > files.txt"), + ShellPolicyDecision::SkipUnsafeAction + ); + } + + #[test] + fn mixed_shell_sequences_are_skipped() { + assert_eq!( + decide("git status; cat src/lib.rs"), + ShellPolicyDecision::SkipMixedShell + ); + assert_eq!( + decide("cat src/lib.rs && git status"), + ShellPolicyDecision::SkipMixedShell + ); + } +} diff --git a/src/protocol.rs b/src/protocol.rs new file mode 100644 index 0000000..7ae7eb3 --- /dev/null +++ b/src/protocol.rs @@ -0,0 +1,376 @@ +use crate::reduce::reduce_execution_with_rules; +use crate::savings::{self, SavingsRecord}; +use crate::types::{ + ClassificationResult, CompactResult, CompiledRule, CompressorKind, ContentKind, ReduceOptions, + ReductionStats, ToolExecutionInput, +}; +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(untagged)] +pub enum ReduceJsonRequest { + Envelope(ReduceJsonEnvelope), + Direct(ToolExecutionInput), +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ReduceJsonEnvelope { + pub input: ToolExecutionInput, + #[serde(default)] + pub options: ReduceOptions, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ReduceJsonResponse { + pub inline_text: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub preview_text: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub facts: Option>, + pub stats: ReductionStats, + pub classification: ClassificationResult, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub metadata: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub trace: Option, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ReduceJsonMetadata { + pub no_omit_requested: bool, + pub store_requested: bool, + pub record_stats_requested: bool, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub ccr: Option, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ReduceJsonCcrRef { + pub token: String, + pub original_chars: usize, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ReduceJsonTrace { + pub tool_name: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub argv0: Option, + pub raw_mode: bool, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub max_inline_chars: Option, + pub family: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub matched_reducer: Option, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ReduceJsonError { + pub code: String, + pub message: String, +} + +impl ReduceJsonError { + fn new(code: &'static str, message: impl Into) -> Self { + Self { + code: code.to_owned(), + message: message.into(), + } + } +} + +pub type ReduceJsonResult = Result; + +pub fn reduce_json_str(json: &str, rules: &[CompiledRule]) -> ReduceJsonResult { + if json.contains('\0') { + return Err(ReduceJsonError::new( + "nul_byte", + "payload contains a NUL byte", + )); + } + let request = serde_json::from_str::(json) + .map_err(|error| ReduceJsonError::new("invalid_json", error.to_string()))?; + reduce_json_request(request, rules) +} + +pub fn reduce_json_request( + request: ReduceJsonRequest, + rules: &[CompiledRule], +) -> ReduceJsonResult { + let (mut input, options) = match request { + ReduceJsonRequest::Envelope(envelope) => (envelope.input, envelope.options), + ReduceJsonRequest::Direct(input) => (input, ReduceOptions::default()), + }; + + if input.cwd.is_none() { + input.cwd = options.cwd.clone(); + } + + let trace_requested = options.trace.unwrap_or(false); + let metadata = metadata_for_options(&options); + let trace_input = trace_requested.then(|| input.clone()); + let result = reduce_execution_with_rules(input, rules, &options); + Ok(response_from_result( + result, + metadata, + trace_input, + &options, + )) +} + +fn response_from_result( + result: CompactResult, + metadata: Option, + trace_input: Option, + options: &ReduceOptions, +) -> ReduceJsonResponse { + if options.record_stats.unwrap_or(false) { + record_stats(&result); + } + + let trace = trace_input.map(|input| { + let argv0 = trace_argv0(&input).map(str::to_owned); + ReduceJsonTrace { + tool_name: input.tool_name, + argv0, + raw_mode: options.raw.unwrap_or(false), + max_inline_chars: options.max_inline_chars, + family: result.classification.family.clone(), + matched_reducer: result.classification.matched_reducer.clone(), + } + }); + + let metadata = metadata.map(|mut metadata| { + if let Some(token) = result.ccr_token.clone() { + metadata.ccr = Some(ReduceJsonCcrRef { + token, + original_chars: result.stats.raw_chars, + }); + } + metadata + }); + + ReduceJsonResponse { + inline_text: result.inline_text, + preview_text: result.preview_text, + facts: result.facts, + stats: result.stats, + classification: result.classification, + metadata, + trace, + } +} + +fn metadata_for_options(options: &ReduceOptions) -> Option { + let metadata = ReduceJsonMetadata { + no_omit_requested: options.no_omit.unwrap_or(false), + store_requested: options.store.unwrap_or(false), + record_stats_requested: options.record_stats.unwrap_or(false), + ccr: None, + }; + (metadata.no_omit_requested || metadata.store_requested || metadata.record_stats_requested) + .then_some(metadata) +} + +fn record_stats(result: &CompactResult) { + let mut record = SavingsRecord::estimated_compaction( + ContentKind::Log, + CompressorKind::Generic, + estimated_tokens_from_chars(result.stats.raw_chars), + estimated_tokens_from_chars(result.stats.reduced_chars), + ) + .with_bytes( + result.stats.raw_chars as u64, + result.stats.reduced_chars as u64, + ) + .with_lossy(result.stats.reduced_chars < result.stats.raw_chars) + .with_ccr_token_present(result.ccr_token.is_some()); + + if let Some(rule_id) = result.classification.matched_reducer.as_deref() { + record = record.with_rule_id(rule_id); + } + + savings::record_event(record); +} + +fn estimated_tokens_from_chars(chars: usize) -> u64 { + if chars == 0 { + 0 + } else { + ((chars as f64) / crate::tokens::CHARS_PER_TOKEN) + .ceil() + .max(1.0) as u64 + } +} + +fn trace_argv0(input: &ToolExecutionInput) -> Option<&str> { + input + .argv + .as_ref() + .and_then(|argv| argv.first().map(String::as_str)) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::rules::load_builtin_rules; + + #[test] + fn reduce_json_accepts_direct_payload() { + let rules = load_builtin_rules(); + let response = reduce_json_str( + r#"{ + "toolName": "bash", + "argv": ["git", "status"], + "stdout": "On branch main\n\nChanges not staged for commit:\n\tmodified: src/lib.rs\n" + }"#, + &rules, + ) + .expect("direct payload"); + + assert_eq!(response.inline_text, "Changes not staged:\nM: src/lib.rs"); + assert_eq!( + response.classification.matched_reducer.as_deref(), + Some("git/status") + ); + assert!(response.metadata.is_none()); + assert!(response.trace.is_none()); + } + + #[test] + fn reduce_json_accepts_envelope_payload_with_options() { + let rules = load_builtin_rules(); + let response = reduce_json_str( + r#"{ + "input": { + "toolName": "bash", + "argv": ["some_tool"], + "stdout": "alpha\nbeta\ngamma\ndelta\nepsilon\nzeta" + }, + "options": { + "maxInlineChars": 24, + "trace": true, + "recordStats": true + } + }"#, + &rules, + ) + .expect("envelope payload"); + + assert!(response.inline_text.len() <= 24, "{response:#?}"); + assert_eq!( + response.metadata, + Some(ReduceJsonMetadata { + no_omit_requested: false, + store_requested: false, + record_stats_requested: true, + ccr: None, + }) + ); + let trace = response.trace.expect("trace"); + assert_eq!(trace.tool_name, "bash"); + assert_eq!(trace.argv0.as_deref(), Some("some_tool")); + assert_eq!(trace.max_inline_chars, Some(24)); + assert_eq!(trace.matched_reducer.as_deref(), Some("generic/fallback")); + } + + #[test] + fn reduce_json_store_option_returns_retrievable_ccr_ref() { + let rules = load_builtin_rules(); + let original = (0..80) + .map(|i| format!("line {i}: verbose report details")) + .collect::>() + .join("\n"); + let response = reduce_json_request( + ReduceJsonRequest::Envelope(ReduceJsonEnvelope { + input: ToolExecutionInput { + tool_name: "bash".to_owned(), + argv: Some(vec!["custom-report".to_owned()]), + stdout: Some(original.clone()), + ..ToolExecutionInput::default() + }, + options: ReduceOptions { + store: Some(true), + max_inline_chars: Some(80), + ..ReduceOptions::default() + }, + }), + &rules, + ) + .expect("stored payload"); + + let metadata = response.metadata.expect("store metadata"); + assert!(metadata.store_requested); + let ccr = metadata.ccr.expect("ccr ref"); + assert_eq!(ccr.original_chars, original.chars().count()); + assert_eq!( + crate::cache::retrieve(&ccr.token).as_deref(), + Some(original.as_str()) + ); + assert!(!response.inline_text.contains(&ccr.token)); + } + + #[test] + fn reduce_json_raw_mode_returns_raw_text() { + let rules = load_builtin_rules(); + let response = reduce_json_str( + r#"{ + "input": { + "toolName": "bash", + "argv": ["git", "status"], + "stdout": "On branch main\n" + }, + "options": { "raw": true } + }"#, + &rules, + ) + .expect("raw payload"); + + assert_eq!(response.inline_text, "On branch main\n"); + } + + #[test] + fn reduce_json_invalid_classifier_falls_back_to_matching() { + let rules = load_builtin_rules(); + let response = reduce_json_str( + r#"{ + "input": { + "toolName": "bash", + "argv": ["git", "status"], + "stdout": "On branch main\n" + }, + "options": { "classifier": "missing/rule" } + }"#, + &rules, + ) + .expect("invalid classifier fallback"); + + assert_eq!( + response.classification.matched_reducer.as_deref(), + Some("git/status") + ); + } + + #[test] + fn reduce_json_rejects_malformed_payload_without_raw_echo() { + let rules = load_builtin_rules(); + let error = reduce_json_str(r#"{"toolName":"bash","stdout":"secret""#, &rules) + .expect_err("invalid json"); + + assert_eq!(error.code, "invalid_json"); + assert!(!error.message.contains("secret")); + } + + #[test] + fn reduce_json_rejects_nul_bytes() { + let rules = load_builtin_rules(); + let error = reduce_json_str("{\"toolName\":\"bash\"}\0", &rules).expect_err("nul byte"); + + assert_eq!(error.code, "nul_byte"); + } +} diff --git a/src/reduce.rs b/src/reduce.rs index 9b1082d..3f7cf58 100644 --- a/src/reduce.rs +++ b/src/reduce.rs @@ -10,6 +10,7 @@ use regex::Regex; use crate::{ classify::classify_execution, + policy::{ShellCompactionPolicy, ShellPolicyDecision, apply_shell_compaction_policy}, text::{ clamp_text, clamp_text_middle, count_text_chars, dedupe_adjacent, head_tail, normalize_lines, pluralize, strip_ansi, trim_empty_edges, @@ -98,21 +99,7 @@ pub fn normalize_execution_input(input: ToolExecutionInput) -> ToolExecutionInpu } } -/// True when the command is a well-known file-content inspection tool. -pub fn is_file_content_inspection_command(input: &ToolExecutionInput) -> bool { - static FILE_TOOLS: &[&str] = &[ - "cat", "sed", "head", "tail", "nl", "bat", "batcat", "jq", "yq", - ]; - let argv = input.argv.as_deref().unwrap_or(&[]); - if argv.is_empty() { - return false; - } - let argv0 = std::path::Path::new(&argv[0]) - .file_name() - .map(|n| n.to_string_lossy().to_string()) - .unwrap_or_default(); - FILE_TOOLS.contains(&argv0.as_str()) -} +pub use crate::policy::is_file_content_inspection_command; // --------------------------------------------------------------------------- // Git-status post-processor @@ -803,13 +790,16 @@ pub fn reduce_execution_with_rules( ratio: 1.0, }, classification, + ccr_token: None, }; } - // File-content inspection commands are never compacted - if classification.matched_reducer.as_deref() == Some("generic/fallback") - && is_file_content_inspection_command(&normalized_input) - { + // Shell output may be summarized only when the host policy says the command + // is safe for compaction. This runs before reducer selection so mixed + // command strings cannot accidentally match a specialised reducer prefix. + let shell_policy_decision = + apply_shell_compaction_policy(&normalized_input, ShellCompactionPolicy::AllowSafeInventory); + if !matches!(shell_policy_decision, ShellPolicyDecision::Compact) { return CompactResult { inline_text: raw_text, preview_text: None, @@ -820,6 +810,7 @@ pub fn reduce_execution_with_rules( ratio: 1.0, }, classification, + ccr_token: None, }; } @@ -870,6 +861,13 @@ pub fn reduce_execution_with_rules( ratio ); + let ccr_token = if opts.store.unwrap_or(false) && reduced_chars < measured_raw_chars { + let (token, retained) = crate::cache::offload_checked(&raw_text); + retained.then_some(token) + } else { + None + }; + CompactResult { inline_text, preview_text: if summary.is_empty() { @@ -884,6 +882,7 @@ pub fn reduce_execution_with_rules( ratio, }, classification, + ccr_token, } } diff --git a/src/reduce_tests.rs b/src/reduce_tests.rs index cf4a392..4d1e28b 100644 --- a/src/reduce_tests.rs +++ b/src/reduce_tests.rs @@ -278,6 +278,58 @@ fn file_inspection_command_with_path_prefix() { assert!(is_file_content_inspection_command(&input)); } +#[test] +fn safe_inventory_pipeline_may_use_generic_fallback() { + let stdout = (0..80) + .map(|i| format!("src/generated/module_{i}.rs")) + .collect::>() + .join("\n"); + let input = ToolExecutionInput { + tool_name: "bash".to_owned(), + command: Some("find . -type f | sort | head -n 20".to_owned()), + stdout: Some(stdout.clone()), + ..Default::default() + }; + let result = run(input); + assert!( + result.inline_text.len() < stdout.len(), + "got: {}", + result.inline_text + ); +} + +#[test] +fn mixed_shell_sequence_stays_raw() { + let stdout = (0..80) + .map(|i| format!("line {i}")) + .collect::>() + .join("\n"); + let input = ToolExecutionInput { + tool_name: "bash".to_owned(), + command: Some("git status; cat src/lib.rs".to_owned()), + stdout: Some(stdout.clone()), + ..Default::default() + }; + let result = run(input); + assert_eq!(result.inline_text, stdout); +} + +#[test] +fn unsafe_find_exec_stays_raw() { + let stdout = (0..80) + .map(|i| format!("file {i} contents")) + .collect::>() + .join("\n"); + let input = ToolExecutionInput { + tool_name: "bash".to_owned(), + command: Some(r"find . -exec cat {} \;".to_owned()), + stdout: Some(stdout.clone()), + ..Default::default() + }; + let result = run(input); + assert_eq!(result.inline_text, stdout); +} + // --- build_raw_text via reduction pipeline --- #[test] diff --git a/src/relevance/bm25.rs b/src/relevance/bm25.rs new file mode 100644 index 0000000..4f07869 --- /dev/null +++ b/src/relevance/bm25.rs @@ -0,0 +1,167 @@ +//! Dependency-free BM25 scorer. + +use std::collections::{HashMap, HashSet}; + +const K1: f32 = 1.2; +const B: f32 = 0.75; + +#[derive(Debug, Clone, PartialEq)] +pub struct Bm25DocumentScore { + pub index: usize, + pub score: f32, +} + +#[derive(Debug, Clone)] +pub struct Bm25Corpus { + docs: Vec>, + frequencies: Vec>, + document_frequency: HashMap, + average_len: f32, +} + +impl Bm25Corpus { + pub fn new<'a>(documents: impl IntoIterator) -> Self { + let docs: Vec> = documents.into_iter().map(tokenize).collect(); + Self::from_tokenized(docs) + } + + pub fn from_tokenized(docs: Vec>) -> Self { + let mut frequencies = Vec::with_capacity(docs.len()); + let mut document_frequency: HashMap = HashMap::new(); + let mut total_len = 0usize; + + for doc in &docs { + total_len += doc.len(); + let mut counts: HashMap = HashMap::new(); + for token in doc { + *counts.entry(token.clone()).or_insert(0) += 1; + } + let unique: HashSet<&String> = doc.iter().collect(); + for token in unique { + *document_frequency.entry(token.clone()).or_insert(0) += 1; + } + frequencies.push(counts); + } + + let average_len = if docs.is_empty() { + 0.0 + } else { + total_len as f32 / docs.len() as f32 + }; + + Self { + docs, + frequencies, + document_frequency, + average_len, + } + } + + pub fn score(&self, query: &str, index: usize) -> f32 { + let Some(doc) = self.docs.get(index) else { + return 0.0; + }; + if doc.is_empty() || self.average_len <= f32::EPSILON { + return 0.0; + } + + let query_tokens = tokenize(query); + if query_tokens.is_empty() { + return 0.0; + } + + let mut score = 0.0f32; + let doc_len = doc.len() as f32; + let doc_count = self.docs.len() as f32; + let counts = &self.frequencies[index]; + let unique_query: HashSet = query_tokens.into_iter().collect(); + + for token in unique_query { + let Some(&tf_count) = counts.get(&token) else { + continue; + }; + let df = *self.document_frequency.get(&token).unwrap_or(&0) as f32; + let idf = ((doc_count - df + 0.5) / (df + 0.5) + 1.0).ln(); + let tf = tf_count as f32; + let denom = tf + K1 * (1.0 - B + B * (doc_len / self.average_len)); + score += idf * (tf * (K1 + 1.0)) / denom; + } + + score + exact_identifier_boost(query, doc) + } + + pub fn score_all(&self, query: &str) -> Vec { + (0..self.docs.len()) + .map(|index| Bm25DocumentScore { + index, + score: self.score(query, index), + }) + .collect() + } +} + +pub fn tokenize(text: &str) -> Vec { + let mut tokens = Vec::new(); + let mut current = String::new(); + + for ch in text.chars() { + if ch.is_ascii_alphanumeric() || ch == '_' || ch == '-' || ch == '.' || ch == '@' { + current.push(ch.to_ascii_lowercase()); + } else { + push_token(&mut tokens, &mut current); + } + } + push_token(&mut tokens, &mut current); + tokens +} + +fn push_token(tokens: &mut Vec, current: &mut String) { + let token = current.trim_matches(|c: char| c == '-' || c == '.' || c == '@'); + if token.len() >= 2 { + tokens.push(token.to_string()); + } + current.clear(); +} + +fn exact_identifier_boost(query: &str, doc: &[String]) -> f32 { + tokenize(query) + .into_iter() + .filter(|token| is_identifier_like(token) && doc.iter().any(|doc_token| doc_token == token)) + .count() as f32 + * 1.5 +} + +fn is_identifier_like(token: &str) -> bool { + token.contains('_') + || token.contains('-') + || token.contains('.') + || token.contains('@') + || token.chars().any(|c| c.is_ascii_digit()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn exact_identifier_match_scores_above_generic_terms() { + let corpus = Bm25Corpus::new([ + "general retry handling around worker state", + "panic in sync_worker_v2 for account id 42", + "worker completed normally after queue drain", + ]); + + let scores = corpus.score_all("sync_worker_v2"); + + assert!(scores[1].score > scores[0].score, "{scores:?}"); + assert!(scores[1].score > scores[2].score, "{scores:?}"); + } + + #[test] + fn tokenization_keeps_dotted_and_email_identifiers() { + let tokens = tokenize("api.worker.example failed for ops@example.com"); + + assert!(tokens.contains(&"api.worker.example".to_string())); + assert!(tokens.contains(&"ops@example.com".to_string())); + } +} diff --git a/src/relevance/mod.rs b/src/relevance/mod.rs new file mode 100644 index 0000000..743bade --- /dev/null +++ b/src/relevance/mod.rs @@ -0,0 +1,5 @@ +//! Shared relevance scoring helpers for compressors and host adapters. + +pub mod bm25; + +pub use bm25::{Bm25Corpus, Bm25DocumentScore, tokenize}; diff --git a/src/rules/builtin.rs b/src/rules/builtin.rs index 159e327..e71bc7f 100644 --- a/src/rules/builtin.rs +++ b/src/rules/builtin.rs @@ -48,6 +48,10 @@ pub static BUILTIN_RULE_JSONS: &[(&str, &str)] = &[ "build/webpack", include_str!("../vendor/rules/build__webpack.json"), ), + ( + "ci/github-actions", + include_str!("../vendor/rules/ci__github-actions.json"), + ), ("cloud/aws", include_str!("../vendor/rules/cloud__aws.json")), ("cloud/az", include_str!("../vendor/rules/cloud__az.json")), ( diff --git a/src/rules/builtin_tests.rs b/src/rules/builtin_tests.rs index 37d6342..ac5e222 100644 --- a/src/rules/builtin_tests.rs +++ b/src/rules/builtin_tests.rs @@ -114,8 +114,8 @@ fn total_builtin_count() { // Update this number when new rules are added. assert_eq!( BUILTIN_RULE_JSONS.len(), - 100, - "expected 100 builtin rules; update this assertion if the vendor set changes" + 101, + "expected 101 builtin rules; update this assertion if the vendor set changes" ); } diff --git a/src/rules/loader.rs b/src/rules/loader.rs index 93917a2..619efed 100644 --- a/src/rules/loader.rs +++ b/src/rules/loader.rs @@ -51,7 +51,7 @@ fn prefer_existing(new: PathBuf, legacy: PathBuf) -> PathBuf { new } -fn user_rules_root(custom: Option<&Path>) -> PathBuf { +pub(crate) fn user_rules_root(custom: Option<&Path>) -> PathBuf { if let Some(p) = custom { return p.to_owned(); } @@ -64,7 +64,7 @@ fn user_rules_root(custom: Option<&Path>) -> PathBuf { ) } -fn project_rules_root(cwd: Option<&Path>, custom: Option<&Path>) -> PathBuf { +pub(crate) fn project_rules_root(cwd: Option<&Path>, custom: Option<&Path>) -> PathBuf { if let Some(p) = custom { return p.to_owned(); } @@ -101,7 +101,7 @@ fn load_builtin_descriptors() -> Vec<(RuleOrigin, String, JsonRule)> { /// Recursively walk `root` and return all `.json` files that are not /// `.schema.json` or `.fixture.json`. -fn list_rule_files(root: &Path) -> Vec { +pub(crate) fn list_rule_files(root: &Path) -> Vec { if !root.is_dir() { return Vec::new(); } diff --git a/src/rules/mod.rs b/src/rules/mod.rs index c73263f..a64981b 100644 --- a/src/rules/mod.rs +++ b/src/rules/mod.rs @@ -3,6 +3,13 @@ pub mod builtin; pub mod compiler; pub mod loader; +pub mod verify; pub use compiler::compile_rule; pub use loader::{LoadRuleOptions, cached_overlay_rules, load_builtin_rules, load_rules}; +pub use verify::{ + RuleDescriptorRef, RuleDiscoveryFamily, RuleDiscoveryReport, RuleDuplicateId, + RuleFixtureFailure, RuleFixtureParseError, RuleFixtureVerificationReport, RuleParseError, + RuleRegexError, RuleShadowedRule, RuleVerificationReport, discover_fallback_outputs, + verify_rule_fixtures, verify_rules, +}; diff --git a/src/rules/verify.rs b/src/rules/verify.rs new file mode 100644 index 0000000..ed46b8f --- /dev/null +++ b/src/rules/verify.rs @@ -0,0 +1,817 @@ +use super::builtin::BUILTIN_RULE_JSONS; +use super::loader::{LoadRuleOptions, list_rule_files, project_rules_root, user_rules_root}; +use crate::classify::classify_execution; +use crate::reduce::reduce_execution_with_rules; +use crate::types::{CompiledRule, JsonRule, RuleFixture, RuleOrigin, ToolExecutionInput}; +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; +use std::collections::HashMap; +use std::path::{Path, PathBuf}; + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct RuleVerificationReport { + pub descriptors_seen: usize, + pub valid_rules: usize, + pub final_rules: usize, + pub parse_errors: Vec, + pub invalid_regexes: Vec, + pub duplicate_ids: Vec, + pub shadowed_rules: Vec, +} + +impl RuleVerificationReport { + pub fn is_clean(&self) -> bool { + self.parse_errors.is_empty() + && self.invalid_regexes.is_empty() + && self.duplicate_ids.is_empty() + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct RuleParseError { + pub source: RuleOrigin, + pub path: String, + pub error: String, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct RuleRegexError { + pub source: RuleOrigin, + pub path: String, + pub rule_id: String, + pub field: String, + pub index: usize, + pub error: String, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct RuleDuplicateId { + pub rule_id: String, + pub occurrences: Vec, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct RuleShadowedRule { + pub rule_id: String, + pub active: RuleDescriptorRef, + pub shadowed: Vec, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct RuleDescriptorRef { + pub source: RuleOrigin, + pub path: String, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct RuleFixtureVerificationReport { + pub fixtures_seen: usize, + pub passed: usize, + pub parse_errors: Vec, + pub failures: Vec, +} + +impl RuleFixtureVerificationReport { + pub fn is_clean(&self) -> bool { + self.parse_errors.is_empty() && self.failures.is_empty() + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct RuleFixtureParseError { + pub path: String, + pub error: String, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct RuleFixtureFailure { + pub path: String, + pub family: String, + #[serde(default)] + pub matched_rule_id: Option, + pub expected_sha256: String, + pub actual_sha256: String, + pub expected_bytes: usize, + pub actual_bytes: usize, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct RuleDiscoveryReport { + pub inputs_seen: usize, + pub fallback_outputs: usize, + pub families: Vec, +} + +impl RuleDiscoveryReport { + pub fn is_empty(&self) -> bool { + self.fallback_outputs == 0 + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct RuleDiscoveryFamily { + pub tool_name: String, + pub argv0: String, + pub count: usize, +} + +#[derive(Debug, Clone)] +struct RuleDescriptor { + source: RuleOrigin, + path: String, + rule: JsonRule, +} + +/// Verify rule roots without changing the lenient loader contract. +pub fn verify_rules(opts: &LoadRuleOptions) -> RuleVerificationReport { + let mut descriptors = Vec::new(); + let mut parse_errors = Vec::new(); + + collect_builtin_descriptors(&mut descriptors, &mut parse_errors); + if !opts.exclude_user { + let root = user_rules_root(opts.user_rules_dir.as_deref()); + collect_disk_descriptors(&root, RuleOrigin::User, &mut descriptors, &mut parse_errors); + } + if !opts.exclude_project { + let root = project_rules_root(opts.cwd.as_deref(), opts.project_rules_dir.as_deref()); + collect_disk_descriptors( + &root, + RuleOrigin::Project, + &mut descriptors, + &mut parse_errors, + ); + } + + let descriptors_seen = descriptors.len() + parse_errors.len(); + let invalid_regexes = collect_regex_errors(&descriptors); + let duplicate_ids = collect_duplicate_ids(&descriptors); + let shadowed_rules = collect_shadowed_rules(&descriptors); + let final_rules = descriptors + .iter() + .map(|descriptor| descriptor.rule.id.as_str()) + .collect::>() + .len(); + + RuleVerificationReport { + descriptors_seen, + valid_rules: descriptors.len(), + final_rules, + parse_errors, + invalid_regexes, + duplicate_ids, + shadowed_rules, + } +} + +/// Verify recorded rule fixtures without exposing raw fixture input or output. +pub fn verify_rule_fixtures( + fixtures_dir: impl AsRef, + rules: &[CompiledRule], +) -> RuleFixtureVerificationReport { + let mut report = RuleFixtureVerificationReport { + fixtures_seen: 0, + passed: 0, + parse_errors: Vec::new(), + failures: Vec::new(), + }; + + for path in list_fixture_files(fixtures_dir.as_ref()) { + report.fixtures_seen += 1; + let path_label = path.display().to_string(); + match read_fixture(&path) { + Ok(fixture) => verify_fixture(&path_label, fixture, rules, &mut report), + Err(error) => report.parse_errors.push(RuleFixtureParseError { + path: path_label, + error, + }), + } + } + + report +} + +/// Count command families that currently fall through to `generic/fallback`. +pub fn discover_fallback_outputs(inputs: I, rules: &[CompiledRule]) -> RuleDiscoveryReport +where + I: IntoIterator, +{ + let mut inputs_seen = 0; + let mut fallback_outputs = 0; + let mut counts: HashMap<(String, String), usize> = HashMap::new(); + + for input in inputs { + inputs_seen += 1; + let classification = classify_execution(&input, rules, None); + if !is_generic_fallback(&classification) { + continue; + } + + fallback_outputs += 1; + let key = discovery_key_for_input(&input); + *counts.entry(key).or_default() += 1; + } + + let mut families = counts + .into_iter() + .map(|((tool_name, argv0), count)| RuleDiscoveryFamily { + tool_name, + argv0, + count, + }) + .collect::>(); + families.sort_by(|a, b| { + b.count + .cmp(&a.count) + .then_with(|| a.tool_name.cmp(&b.tool_name)) + .then_with(|| a.argv0.cmp(&b.argv0)) + }); + + RuleDiscoveryReport { + inputs_seen, + fallback_outputs, + families, + } +} + +fn is_generic_fallback(classification: &crate::types::ClassificationResult) -> bool { + classification.matched_reducer.as_deref() == Some("generic/fallback") + || (classification.family == "generic" && classification.matched_reducer.is_none()) +} + +fn discovery_key_for_input(input: &ToolExecutionInput) -> (String, String) { + ( + sanitize_discovery_label(&input.tool_name), + discovery_argv0(input) + .map(sanitize_discovery_label) + .unwrap_or_else(|| "unknown".to_owned()), + ) +} + +fn discovery_argv0(input: &ToolExecutionInput) -> Option<&str> { + input + .argv + .as_ref() + .and_then(|argv| argv.first().map(String::as_str)) + .or_else(|| { + input + .command + .as_deref() + .and_then(|command| command.split_whitespace().next()) + }) +} + +fn sanitize_discovery_label(raw: &str) -> String { + let trimmed = raw.trim_matches(|ch: char| ch.is_whitespace() || ch == '"' || ch == '\''); + let basename = trimmed.rsplit(['/', '\\']).next().unwrap_or(trimmed); + let label = basename + .chars() + .filter(|ch| ch.is_ascii_alphanumeric() || matches!(ch, '.' | '_' | '-' | '+' | ':')) + .take(80) + .collect::(); + if label.is_empty() { + "unknown".to_owned() + } else { + label + } +} + +fn list_fixture_files(root: &Path) -> Vec { + let mut out = Vec::new(); + collect_fixture_files(root, &mut out); + out.sort(); + out +} + +fn collect_fixture_files(root: &Path, out: &mut Vec) { + let Ok(entries) = std::fs::read_dir(root) else { + return; + }; + + for entry in entries.flatten() { + let path = entry.path(); + if path.is_dir() { + collect_fixture_files(&path, out); + } else if path + .file_name() + .and_then(|name| name.to_str()) + .is_some_and(|name| name.ends_with(".fixture.json")) + { + out.push(path); + } + } +} + +fn read_fixture(path: &Path) -> Result { + let raw = std::fs::read_to_string(path).map_err(|error| error.to_string())?; + serde_json::from_str(&raw).map_err(|error| error.to_string()) +} + +fn verify_fixture( + path: &str, + fixture: RuleFixture, + rules: &[CompiledRule], + report: &mut RuleFixtureVerificationReport, +) { + let expected = fixture.expected_output.trim_end(); + let options = fixture.options.unwrap_or_default(); + let result = reduce_execution_with_rules(fixture.input, rules, &options); + let actual = result.inline_text.trim_end(); + + if actual == expected { + report.passed += 1; + return; + } + + report.failures.push(RuleFixtureFailure { + path: path.to_owned(), + family: result.classification.family, + matched_rule_id: result.classification.matched_reducer, + expected_sha256: sha256_hex(expected), + actual_sha256: sha256_hex(actual), + expected_bytes: expected.len(), + actual_bytes: actual.len(), + }); +} + +fn sha256_hex(text: &str) -> String { + let mut hasher = Sha256::new(); + hasher.update(text.as_bytes()); + hex::encode(hasher.finalize()) +} + +fn collect_builtin_descriptors( + descriptors: &mut Vec, + parse_errors: &mut Vec, +) { + for (id, json) in BUILTIN_RULE_JSONS { + let path = format!("builtin:{id}"); + match serde_json::from_str::(json) { + Ok(rule) => descriptors.push(RuleDescriptor { + source: RuleOrigin::Builtin, + path, + rule, + }), + Err(error) => parse_errors.push(RuleParseError { + source: RuleOrigin::Builtin, + path, + error: error.to_string(), + }), + } + } +} + +fn collect_disk_descriptors( + root: &Path, + source: RuleOrigin, + descriptors: &mut Vec, + parse_errors: &mut Vec, +) { + for path in list_rule_files(root) { + let path_label = path.display().to_string(); + match std::fs::read_to_string(&path) { + Ok(json) => match serde_json::from_str::(&json) { + Ok(rule) => descriptors.push(RuleDescriptor { + source: source.clone(), + path: path_label, + rule, + }), + Err(error) => parse_errors.push(RuleParseError { + source: source.clone(), + path: path_label, + error: error.to_string(), + }), + }, + Err(error) => parse_errors.push(RuleParseError { + source: source.clone(), + path: path_label, + error: error.to_string(), + }), + } + } +} + +fn collect_regex_errors(descriptors: &[RuleDescriptor]) -> Vec { + let mut out = Vec::new(); + for descriptor in descriptors { + if let Some(filters) = &descriptor.rule.filters { + if let Some(patterns) = &filters.skip_patterns { + collect_pattern_errors( + descriptor, + "filters.skipPatterns", + patterns, + None, + &mut out, + ); + } + if let Some(patterns) = &filters.keep_patterns { + collect_pattern_errors( + descriptor, + "filters.keepPatterns", + patterns, + None, + &mut out, + ); + } + } + if let Some(counters) = &descriptor.rule.counters { + for (index, counter) in counters.iter().enumerate() { + collect_single_regex_error( + descriptor, + "counters.pattern", + index, + &counter.pattern, + counter.flags.as_deref(), + &mut out, + ); + } + } + if let Some(entries) = &descriptor.rule.match_output { + for (index, entry) in entries.iter().enumerate() { + collect_single_regex_error( + descriptor, + "matchOutput.pattern", + index, + &entry.pattern, + entry.flags.as_deref(), + &mut out, + ); + } + } + } + out +} + +fn collect_pattern_errors( + descriptor: &RuleDescriptor, + field: &'static str, + patterns: &[String], + flags: Option<&str>, + out: &mut Vec, +) { + for (index, pattern) in patterns.iter().enumerate() { + collect_single_regex_error(descriptor, field, index, pattern, flags, out); + } +} + +fn collect_single_regex_error( + descriptor: &RuleDescriptor, + field: &'static str, + index: usize, + pattern: &str, + flags: Option<&str>, + out: &mut Vec, +) { + if let Err(error) = build_rule_regex(pattern, flags) { + out.push(RuleRegexError { + source: descriptor.source.clone(), + path: descriptor.path.clone(), + rule_id: descriptor.rule.id.clone(), + field: field.to_owned(), + index, + error, + }); + } +} + +fn build_rule_regex(pattern: &str, flags: Option<&str>) -> Result { + let case_insensitive = flags.map(|f| f.contains('i')).unwrap_or(false); + let multiline = flags.map(|f| f.contains('m')).unwrap_or(false); + let prefix = match (case_insensitive, multiline) { + (true, true) => "(?im)", + (true, false) => "(?i)", + (false, true) => "(?m)", + (false, false) => "", + }; + regex::Regex::new(&format!("{prefix}{pattern}")).map_err(|error| error.to_string()) +} + +fn collect_duplicate_ids(descriptors: &[RuleDescriptor]) -> Vec { + let mut by_id: HashMap<&str, Vec> = HashMap::new(); + for descriptor in descriptors { + by_id + .entry(descriptor.rule.id.as_str()) + .or_default() + .push(ref_for_descriptor(descriptor)); + } + let mut duplicates = by_id + .into_iter() + .filter_map(|(rule_id, occurrences)| { + (occurrences.len() > 1).then(|| RuleDuplicateId { + rule_id: rule_id.to_owned(), + occurrences, + }) + }) + .collect::>(); + duplicates.sort_by(|a, b| a.rule_id.cmp(&b.rule_id)); + duplicates +} + +fn collect_shadowed_rules(descriptors: &[RuleDescriptor]) -> Vec { + let mut by_id: HashMap<&str, Vec<&RuleDescriptor>> = HashMap::new(); + for descriptor in descriptors { + by_id + .entry(descriptor.rule.id.as_str()) + .or_default() + .push(descriptor); + } + let mut shadowed = by_id + .into_iter() + .filter_map(|(rule_id, occurrences)| { + let (active, shadowed) = occurrences.split_last()?; + (!shadowed.is_empty()).then(|| RuleShadowedRule { + rule_id: rule_id.to_owned(), + active: ref_for_descriptor(active), + shadowed: shadowed + .iter() + .map(|descriptor| ref_for_descriptor(descriptor)) + .collect(), + }) + }) + .collect::>(); + shadowed.sort_by(|a, b| a.rule_id.cmp(&b.rule_id)); + shadowed +} + +fn ref_for_descriptor(descriptor: &RuleDescriptor) -> RuleDescriptorRef { + RuleDescriptorRef { + source: descriptor.source.clone(), + path: descriptor.path.clone(), + } +} + +#[cfg(test)] +mod tests { + use super::super::load_builtin_rules; + use super::*; + + fn write_rule(dir: &Path, filename: &str, json: &str) { + std::fs::write(dir.join(filename), json).expect("write rule"); + } + + fn write_fixture(dir: &Path, filename: &str, json: &str) { + std::fs::write(dir.join(filename), json).expect("write fixture"); + } + + #[test] + fn builtin_rules_verify_cleanly() { + let report = verify_rules(&LoadRuleOptions { + exclude_user: true, + exclude_project: true, + ..Default::default() + }); + + assert_eq!(report.descriptors_seen, 101); + assert_eq!(report.valid_rules, 101); + assert_eq!(report.final_rules, 101); + assert!(report.parse_errors.is_empty(), "{report:#?}"); + assert!(report.duplicate_ids.is_empty(), "{report:#?}"); + assert_eq!(report.invalid_regexes.len(), 11, "{report:#?}"); + assert!( + report + .invalid_regexes + .iter() + .any(|error| error.rule_id == "filesystem/find") + ); + assert!(report.shadowed_rules.is_empty()); + } + + #[test] + fn verifier_reports_parse_and_regex_errors() { + let dir = tempfile::tempdir().expect("tempdir"); + write_rule(dir.path(), "bad.json", "{ not json }"); + write_rule( + dir.path(), + "bad_regex.json", + r#"{ + "id": "test/bad-regex", + "family": "test", + "match": {}, + "filters": { "skipPatterns": ["[invalid"] }, + "counters": [{ "name": "bad", "pattern": "(unclosed" }], + "matchOutput": [{ "pattern": "[bad", "message": "ignored" }] + }"#, + ); + + let report = verify_rules(&LoadRuleOptions { + project_rules_dir: Some(dir.path().to_owned()), + exclude_user: true, + ..Default::default() + }); + + assert_eq!(report.parse_errors.len(), 1); + assert_eq!(report.parse_errors[0].source, RuleOrigin::Project); + let project_regex_errors = report + .invalid_regexes + .iter() + .filter(|error| error.source == RuleOrigin::Project) + .collect::>(); + assert_eq!(project_regex_errors.len(), 3); + assert!( + project_regex_errors + .iter() + .any(|error| error.field == "filters.skipPatterns") + ); + assert!( + project_regex_errors + .iter() + .any(|error| error.field == "counters.pattern") + ); + assert!( + project_regex_errors + .iter() + .any(|error| error.field == "matchOutput.pattern") + ); + } + + #[test] + fn verifier_reports_duplicate_and_shadowed_rules() { + let user_dir = tempfile::tempdir().expect("user tempdir"); + let project_dir = tempfile::tempdir().expect("project tempdir"); + write_rule( + user_dir.path(), + "a.json", + r#"{"id":"git/status","family":"user","match":{}}"#, + ); + write_rule( + project_dir.path(), + "b.json", + r#"{"id":"git/status","family":"project","match":{}}"#, + ); + + let report = verify_rules(&LoadRuleOptions { + user_rules_dir: Some(user_dir.path().to_owned()), + project_rules_dir: Some(project_dir.path().to_owned()), + ..Default::default() + }); + + let duplicate = report + .duplicate_ids + .iter() + .find(|duplicate| duplicate.rule_id == "git/status") + .expect("git/status duplicate reported"); + assert_eq!(duplicate.occurrences.len(), 3); + + let shadowed = report + .shadowed_rules + .iter() + .find(|shadowed| shadowed.rule_id == "git/status") + .expect("git/status shadowing reported"); + assert_eq!(shadowed.active.source, RuleOrigin::Project); + assert_eq!(shadowed.shadowed.len(), 2); + } + + #[test] + fn fixture_verifier_reports_existing_fixtures_pass() { + let fixtures_dir = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures"); + assert!( + fixtures_dir.is_dir(), + "missing fixture directory {}", + fixtures_dir.display() + ); + + let rules = load_builtin_rules(); + let report = verify_rule_fixtures(&fixtures_dir, &rules); + + assert_eq!(report.fixtures_seen, 5, "{report:#?}"); + assert_eq!(report.passed, 5, "{report:#?}"); + assert!(report.parse_errors.is_empty(), "{report:#?}"); + assert!(report.failures.is_empty(), "{report:#?}"); + assert!(report.is_clean()); + } + + #[test] + fn fixture_verifier_reports_hash_only_mismatch() { + let dir = tempfile::tempdir().expect("tempdir"); + write_fixture( + dir.path(), + "mismatch.fixture.json", + r#"{ + "description": "non-sensitive mismatch marker", + "input": { + "toolName": "bash", + "argv": ["git", "status"], + "stdout": "On branch main\n\nChanges not staged for commit:\n\tmodified: src/foo.rs\n" + }, + "expectedOutput": "wrong compact text" + }"#, + ); + + let rules = load_builtin_rules(); + let report = verify_rule_fixtures(dir.path(), &rules); + + assert_eq!(report.fixtures_seen, 1, "{report:#?}"); + assert_eq!(report.passed, 0, "{report:#?}"); + assert!(report.parse_errors.is_empty(), "{report:#?}"); + assert_eq!(report.failures.len(), 1, "{report:#?}"); + + let failure = &report.failures[0]; + assert_eq!(failure.family, "git-status"); + assert_eq!(failure.matched_rule_id.as_deref(), Some("git/status")); + assert_ne!(failure.expected_sha256, failure.actual_sha256); + assert_eq!(failure.expected_bytes, "wrong compact text".len()); + + let diagnostic = format!("{failure:?}"); + assert!(!diagnostic.contains("wrong compact text")); + assert!(!diagnostic.contains("On branch main")); + assert!(!diagnostic.contains("src/foo.rs")); + } + + #[test] + fn fixture_verifier_reports_parse_errors() { + let dir = tempfile::tempdir().expect("tempdir"); + write_fixture(dir.path(), "bad.fixture.json", "{ not json }"); + + let rules = load_builtin_rules(); + let report = verify_rule_fixtures(dir.path(), &rules); + + assert_eq!(report.fixtures_seen, 1, "{report:#?}"); + assert_eq!(report.passed, 0, "{report:#?}"); + assert_eq!(report.parse_errors.len(), 1, "{report:#?}"); + assert!(report.failures.is_empty(), "{report:#?}"); + assert!(!report.is_clean()); + } + + #[test] + fn discovery_groups_generic_fallback_outputs_by_command_family() { + let rules = load_builtin_rules(); + let inputs = vec![ + ToolExecutionInput { + tool_name: "bash".to_owned(), + argv: Some(vec!["unknown-build".to_owned(), "--json".to_owned()]), + stdout: Some("secret output should not be reported".to_owned()), + ..Default::default() + }, + ToolExecutionInput { + tool_name: "bash".to_owned(), + command: Some("/tmp/work/unknown-build --again".to_owned()), + stderr: Some("another secret output".to_owned()), + ..Default::default() + }, + ToolExecutionInput { + tool_name: "bash".to_owned(), + argv: Some(vec!["git".to_owned(), "status".to_owned()]), + stdout: Some("On branch main".to_owned()), + ..Default::default() + }, + ]; + + let report = discover_fallback_outputs(inputs, &rules); + + assert_eq!(report.inputs_seen, 3, "{report:#?}"); + assert_eq!(report.fallback_outputs, 2, "{report:#?}"); + assert_eq!(report.families.len(), 1, "{report:#?}"); + assert_eq!(report.families[0].tool_name, "bash"); + assert_eq!(report.families[0].argv0, "unknown-build"); + assert_eq!(report.families[0].count, 2); + assert!(!report.is_empty()); + + let diagnostic = format!("{report:?}"); + assert!(!diagnostic.contains("secret output")); + assert!(!diagnostic.contains("/tmp/work")); + assert!(!diagnostic.contains("On branch main")); + } + + #[test] + fn discovery_sorts_fallback_families_by_count_then_label() { + let rules = load_builtin_rules(); + let inputs = vec![ + ToolExecutionInput { + tool_name: "bash".to_owned(), + argv: Some(vec!["beta".to_owned()]), + ..Default::default() + }, + ToolExecutionInput { + tool_name: "bash".to_owned(), + argv: Some(vec!["alpha".to_owned()]), + ..Default::default() + }, + ToolExecutionInput { + tool_name: "bash".to_owned(), + argv: Some(vec!["beta".to_owned(), "again".to_owned()]), + ..Default::default() + }, + ]; + + let report = discover_fallback_outputs(inputs, &rules); + + assert_eq!( + report + .families + .iter() + .map(|family| (family.argv0.as_str(), family.count)) + .collect::>(), + vec![("beta", 2), ("alpha", 1)] + ); + } +} diff --git a/src/savings.rs b/src/savings.rs index d36fc91..37faada 100644 --- a/src/savings.rs +++ b/src/savings.rs @@ -8,18 +8,168 @@ use std::sync::{Arc, OnceLock, RwLock}; use crate::types::{CompressorKind, ContentKind}; +/// Accounting confidence for a savings event. +/// +/// The class describes the numbers in the record, not the compressor itself: +/// reducer byte counts are measured, token counts from [`crate::tokens`] are +/// estimates, and call-collapse events are counted. +#[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum AccountingClass { + Counted, + Measured, + Estimated, +} + +impl AccountingClass { + pub fn as_str(self) -> &'static str { + match self { + Self::Counted => "counted", + Self::Measured => "measured", + Self::Estimated => "estimated", + } + } +} + +/// Origin of a savings record. +/// +/// Live runtime observations and offline fixture benchmark results must stay +/// distinguishable so dashboards never present benchmark outcomes as live +/// production savings. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, serde::Serialize, serde::Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum SavingsRecordSource { + #[default] + Live, + FixtureBenchmark, +} + +impl SavingsRecordSource { + pub fn as_str(self) -> &'static str { + match self { + Self::Live => "live", + Self::FixtureBenchmark => "fixture_benchmark", + } + } +} + +/// Provider-reported usage fields hosts may attach when real model usage is +/// available. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, serde::Serialize, serde::Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ModelUsage { + pub input_tokens: u64, + pub output_tokens: u64, + pub cache_read_tokens: u64, + pub cache_creation_tokens: u64, +} + +/// Non-sensitive attribution record for a TokenJuice savings event. +#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct SavingsRecord { + pub accounting_class: AccountingClass, + #[serde(default)] + pub source: SavingsRecordSource, + pub content_kind: ContentKind, + pub compressor: CompressorKind, + pub original_tokens: u64, + pub compacted_tokens: u64, + pub original_bytes: Option, + pub compacted_bytes: Option, + pub lossy: bool, + pub ccr_token_present: bool, + pub rule_id: Option, + pub skip_reason: Option, + pub measured_usage: Option, +} + +impl SavingsRecord { + pub fn estimated_compaction( + content_kind: ContentKind, + compressor: CompressorKind, + original_tokens: u64, + compacted_tokens: u64, + ) -> Self { + Self { + accounting_class: AccountingClass::Estimated, + source: SavingsRecordSource::Live, + content_kind, + compressor, + original_tokens, + compacted_tokens, + original_bytes: None, + compacted_bytes: None, + lossy: false, + ccr_token_present: false, + rule_id: None, + skip_reason: None, + measured_usage: None, + } + } + + pub fn with_source(mut self, source: SavingsRecordSource) -> Self { + self.source = source; + self + } + + pub fn with_bytes(mut self, original_bytes: u64, compacted_bytes: u64) -> Self { + self.original_bytes = Some(original_bytes); + self.compacted_bytes = Some(compacted_bytes); + self + } + + pub fn with_lossy(mut self, lossy: bool) -> Self { + self.lossy = lossy; + self + } + + pub fn with_ccr_token_present(mut self, ccr_token_present: bool) -> Self { + self.ccr_token_present = ccr_token_present; + self + } + + pub fn with_rule_id(mut self, rule_id: impl Into) -> Self { + self.rule_id = Some(rule_id.into()); + self + } + + pub fn with_skip_reason(mut self, skip_reason: impl Into) -> Self { + self.skip_reason = Some(skip_reason.into()); + self + } + + pub fn with_measured_usage(mut self, measured_usage: ModelUsage) -> Self { + self.measured_usage = Some(measured_usage); + self + } +} + pub type SavingsRecorder = dyn Fn(ContentKind, CompressorKind, u64, u64) + Send + Sync + 'static; +pub type SavingsRecordRecorder = dyn Fn(SavingsRecord) + Send + Sync + 'static; fn recorder_cell() -> &'static RwLock>> { static RECORDER: OnceLock>>> = OnceLock::new(); RECORDER.get_or_init(|| RwLock::new(None)) } +fn record_recorder_cell() -> &'static RwLock>> { + static RECORDER: OnceLock>>> = OnceLock::new(); + RECORDER.get_or_init(|| RwLock::new(None)) +} + /// Install or clear the host savings recorder. pub fn configure_recorder(recorder: Option>) { *recorder_cell().write().unwrap_or_else(|p| p.into_inner()) = recorder; } +/// Install or clear the host recorder for rich, class-labeled savings records. +pub fn configure_record_recorder(recorder: Option>) { + *record_recorder_cell() + .write() + .unwrap_or_else(|p| p.into_inner()) = recorder; +} + /// Record one compaction event. No-op when no host recorder is installed. pub fn record( content_kind: ContentKind, @@ -27,20 +177,135 @@ pub fn record( original_tokens: u64, compacted_tokens: u64, ) { + record_event(SavingsRecord::estimated_compaction( + content_kind, + compressor, + original_tokens, + compacted_tokens, + )); +} + +/// Record one rich savings event. No-op when no host recorder is installed. +pub fn record_event(record: SavingsRecord) { + let record_recorder = record_recorder_cell() + .read() + .unwrap_or_else(|p| p.into_inner()) + .clone(); + if let Some(recorder) = record_recorder { + recorder(record.clone()); + } + let recorder = recorder_cell() .read() .unwrap_or_else(|p| p.into_inner()) .clone(); if let Some(recorder) = recorder { - recorder(content_kind, compressor, original_tokens, compacted_tokens); + recorder( + record.content_kind, + record.compressor, + record.original_tokens, + record.compacted_tokens, + ); } } #[cfg(test)] mod tests { - use super::*; use std::sync::{Arc, Mutex}; + use super::*; + + #[test] + fn rich_record_serializes_only_metadata() { + let record = + SavingsRecord::estimated_compaction(ContentKind::Log, CompressorKind::Log, 120, 40) + .with_bytes(480, 160) + .with_lossy(true) + .with_ccr_token_present(true) + .with_rule_id("cargo-test") + .with_skip_reason("no_raw_content_here") + .with_measured_usage(ModelUsage { + input_tokens: 10, + output_tokens: 20, + cache_read_tokens: 5, + cache_creation_tokens: 7, + }); + + let json = serde_json::to_string(&record).expect("record serializes"); + assert!(json.contains("\"accountingClass\":\"estimated\"")); + assert!(json.contains("\"source\":\"live\"")); + assert!(json.contains("\"contentKind\":\"log\"")); + assert!(json.contains("\"compressor\":\"log\"")); + assert!(!json.contains("secret tool output")); + assert!(!json.contains("prompt")); + } + + #[test] + fn fixture_benchmark_records_are_labeled_separately() { + let record = SavingsRecord::estimated_compaction( + ContentKind::Json, + CompressorKind::SmartCrusher, + 80, + 30, + ) + .with_source(SavingsRecordSource::FixtureBenchmark) + .with_bytes(320, 120); + + let json = serde_json::to_string(&record).expect("record serializes"); + assert!(json.contains("\"source\":\"fixture_benchmark\"")); + assert!(!json.contains("fixture raw content")); + } + + #[test] + fn record_event_invokes_rich_and_legacy_recorders() { + let rich_records = Arc::new(Mutex::new(Vec::new())); + let legacy_records = Arc::new(Mutex::new(Vec::new())); + + let rich_records_clone = Arc::clone(&rich_records); + configure_record_recorder(Some(Arc::new(move |record| { + rich_records_clone + .lock() + .expect("rich records lock") + .push(record); + }))); + + let legacy_records_clone = Arc::clone(&legacy_records); + configure_recorder(Some(Arc::new( + move |kind, compressor, original, compacted| { + legacy_records_clone + .lock() + .expect("legacy records lock") + .push((kind, compressor, original, compacted)); + }, + ))); + + record_event( + SavingsRecord::estimated_compaction( + ContentKind::Json, + CompressorKind::SmartCrusher, + 100, + 25, + ) + .with_bytes(400, 100) + .with_ccr_token_present(true), + ); + + configure_record_recorder(None); + configure_recorder(None); + + let rich = rich_records.lock().expect("rich records lock"); + assert_eq!(rich.len(), 1); + assert_eq!(rich[0].accounting_class, AccountingClass::Estimated); + assert_eq!(rich[0].original_bytes, Some(400)); + assert!(rich[0].ccr_token_present); + + let legacy = legacy_records.lock().expect("legacy records lock"); + assert_eq!( + legacy.as_slice(), + &[(ContentKind::Json, CompressorKind::SmartCrusher, 100, 25)] + ); + } + #[test] fn no_recorder_is_a_noop() { configure_recorder(None); diff --git a/src/tool_integration.rs b/src/tool_integration.rs index bbd3d0d..406deb5 100644 --- a/src/tool_integration.rs +++ b/src/tool_integration.rs @@ -52,12 +52,11 @@ pub fn current_options() -> CompressOptions { .clone() } -fn options_for_agent(profile: AgentTokenjuiceCompression) -> Option { +fn options_for_agent(profile: AgentTokenjuiceCompression) -> Result { match profile { - AgentTokenjuiceCompression::Off => None, - AgentTokenjuiceCompression::Auto | AgentTokenjuiceCompression::Full => { - Some(current_options()) - } + AgentTokenjuiceCompression::Off => Err("none/agent-profile-off"), + AgentTokenjuiceCompression::Auto => Err("none/agent-profile-auto-unresolved"), + AgentTokenjuiceCompression::Full => Ok(current_options()), AgentTokenjuiceCompression::Light => { let mut opts = current_options(); // Coding agents need raw, exact tool text more than aggressive @@ -67,7 +66,7 @@ fn options_for_agent(profile: AgentTokenjuiceCompression) -> Option (String, CompactionStats) { let original_bytes = output.len(); - let Some(opts) = options_for_agent(profile) else { - log::debug!( - "[tinyjuice] agent profile disabled compaction tool={} bytes={}", - tool_name, - original_bytes - ); - return ( - output.to_string(), - CompactionStats { - tool_name: tool_name.to_string(), - original_bytes, - compacted_bytes: original_bytes, - rule_id: "none/agent-profile-off".to_string(), - applied: false, - }, - ); + let opts = match options_for_agent(profile) { + Ok(opts) => opts, + Err(rule_id) => { + log::debug!( + "[tinyjuice] agent profile skipped compaction tool={} profile={} bytes={}", + tool_name, + profile.as_str(), + original_bytes + ); + return ( + output.to_string(), + CompactionStats { + tool_name: tool_name.to_string(), + original_bytes, + compacted_bytes: original_bytes, + rule_id: rule_id.to_string(), + applied: false, + }, + ); + } }; // A recovery tool's output is the original we previously offloaded — never @@ -296,6 +299,23 @@ mod tests { use super::*; use serde_json::json; + fn stringified_json_rows() -> String { + let rows: Vec<_> = (0..80) + .map(|i| { + json!({ + "id": i, + "status": "active", + "metadata": json!({ + "owner": format!("team-{i}"), + "flags": { "retry": i % 2 == 0 } + }) + .to_string() + }) + }) + .collect(); + serde_json::to_string_pretty(&rows).expect("rows serialize") + } + #[tokio::test] async fn skips_short_output() { let (out, stats) = compact_tool_output_with_policy( @@ -329,6 +349,37 @@ mod tests { .await; assert!(stats.applied, "expected compaction, got {:?}", stats); assert!(compacted.len() < output.len()); + assert!( + compacted.contains("M: src/file_0.rs"), + "git/status rule should rewrite modified paths: {compacted}" + ); + assert!( + !compacted.contains("On branch main"), + "git/status rule should remove branch boilerplate: {compacted}" + ); + } + + #[tokio::test] + async fn compacts_stringified_json_through_tool_adapter() { + let output = stringified_json_rows(); + let args = json!({"path": "accounts.json"}); + + let (compacted, stats) = compact_tool_output_with_policy( + "web_fetch", + Some(&args), + &output, + Some(0), + AgentTokenjuiceCompression::Full, + ) + .await; + + assert!(stats.applied, "expected SmartCrusher, got {:?}", stats); + assert_eq!(stats.rule_id, "smartcrusher"); + assert!(compacted.contains("metadata.owner"), "{compacted}"); + assert!(compacted.contains("metadata.flags.retry"), "{compacted}"); + assert!(compacted.contains("team-7"), "{compacted}"); + assert!(compacted.contains("tinyjuice_retrieve"), "{compacted}"); + assert!(compacted.len() < output.len()); } #[tokio::test] @@ -385,6 +436,28 @@ mod tests { assert_eq!(returned, big); } + #[tokio::test] + async fn auto_agent_profile_requires_host_resolution() { + let mut lines = vec!["On branch main".to_owned()]; + for i in 0..200 { + lines.push(format!("\tmodified: src/file_{i}.rs")); + } + let output = lines.join("\n"); + let args = json!({"command": "git status"}); + let (returned, stats) = compact_tool_output_with_policy( + "shell", + Some(&args), + &output, + Some(0), + AgentTokenjuiceCompression::Auto, + ) + .await; + + assert_eq!(returned, output); + assert!(!stats.applied); + assert_eq!(stats.rule_id, "none/agent-profile-auto-unresolved"); + } + #[test] fn extract_argv_handles_common_shapes() { let (cmd, argv) = extract_command_argv(Some(&json!({"command": "git status"}))); diff --git a/src/types.rs b/src/types.rs index 7e45ae9..fa94c66 100644 --- a/src/types.rs +++ b/src/types.rs @@ -260,6 +260,21 @@ pub struct ReduceOptions { /// Working directory for project-layer rule discovery. #[serde(default)] pub cwd: Option, + /// Protocol compatibility flag for callers that want no lossy omission. + #[serde(default)] + pub no_omit: Option, + /// Request artifact/CCR storage when a protocol surface supports it. + #[serde(default)] + pub store: Option, + /// Storage directory for protocol surfaces that support artifacts. + #[serde(default)] + pub store_dir: Option, + /// Request metadata-only reducer trace fields in protocol responses. + #[serde(default)] + pub trace: Option, + /// Request metadata-only stats recording in protocol adapters. + #[serde(default)] + pub record_stats: Option, } // --------------------------------------------------------------------------- @@ -299,13 +314,17 @@ pub struct CompactResult { pub facts: Option>, pub stats: ReductionStats, pub classification: ClassificationResult, + /// CCR token for the original raw output when a protocol surface requested + /// storage and the store retained the content. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub ccr_token: Option, } /// Per-agent TokenJuice profile. /// -/// `Auto` is resolved by the agent definition layer. TokenJuice itself treats -/// `Auto` like `Full` so non-agent callers keep the global `[tinyjuice]` -/// behaviour unless they explicitly pass a narrower profile. +/// `Auto` must be resolved by the host agent definition layer before calling +/// TokenJuice. The compression adapter treats unresolved `Auto` as a +/// passthrough so hosts cannot accidentally apply `Full` to coding agents. #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)] #[serde(rename_all = "snake_case")] pub enum AgentTokenjuiceCompression { @@ -393,6 +412,193 @@ pub struct ContentHint { pub query: Option, /// Hard override — when set, detection returns this kind verbatim. pub explicit: Option, + /// Whether file-read content must remain exact or may be reduced to a code + /// stub. Defaults to exact; hosts must opt into stubbing explicitly. + pub read_intent: ReadIntent, +} + +/// Caller intent for read-like sources. +#[derive(Debug, Clone, PartialEq, Eq, Default)] +pub enum ReadIntent { + /// Preserve bytes exactly. This is the default for file reads. + #[default] + Exact, + /// Return a structural source-code stub. + Stub(StubMode), +} + +/// Source-code stub mode. +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", tag = "kind", content = "value")] +pub enum StubMode { + #[default] + SignaturesOnly, + PublicApi, + MatchedSymbols(Vec), + ExpandAroundLines(Vec), +} + +/// One-based inclusive line range. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct LineRange { + pub start: usize, + pub end: usize, +} + +impl LineRange { + pub fn new(start: usize, end: usize) -> Self { + Self { + start: start.max(1), + end: end.max(start.max(1)), + } + } + + pub fn intersects(self, other: Self) -> bool { + self.start <= other.end && other.start <= self.end + } +} + +/// Parser path used for a source-code stub. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum ParseStatus { + TreeSitter, + HeuristicFallback, +} + +/// Symbol surfaced in a source-code stub. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct SymbolSummary { + pub name: String, + pub kind: String, + pub start_line: usize, + pub end_line: usize, + pub public: bool, +} + +/// Source range omitted from a stub. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct CodeElision { + pub start_line: usize, + pub end_line: usize, + pub reason: String, +} + +/// Structured source-code stub result. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct CodeStubOutput { + pub text: String, + pub symbols: Vec, + pub elisions: Vec, + pub parse_status: ParseStatus, +} + +// --------------------------------------------------------------------------- +// Web extraction reducer inputs +// --------------------------------------------------------------------------- + +/// Format of already-extracted web content handed to the web reducer. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum WebExtractFormat { + #[default] + Markdown, + Text, + Html, +} + +impl WebExtractFormat { + pub fn as_str(self) -> &'static str { + match self { + Self::Markdown => "markdown", + Self::Text => "text", + Self::Html => "html", + } + } +} + +/// One already-extracted web page to reduce. Hosts own fetching and URL +/// validation; TinyJuice receives only extracted content plus metadata. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct WebExtractReduceInput { + pub url: String, + #[serde(default)] + pub title: Option, + pub content: String, + #[serde(default)] + pub format: WebExtractFormat, + #[serde(default)] + pub provider: Option, + #[serde(default)] + pub char_limit: Option, + #[serde(default)] + pub metadata: serde_json::Map, +} + +/// Batch shape for multi-URL extractors. The reducer keeps page footers intact +/// even when the combined output exceeds the inline budget. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct WebExtractBatchInput { + pub pages: Vec, + #[serde(default)] + pub default_char_limit: Option, + #[serde(default)] + pub max_combined_inline_chars: Option, +} + +/// Web extraction truncation knobs. Defaults match the P1-2 plan. +#[derive(Debug, Clone, Copy, PartialEq)] +pub struct WebExtractOptions { + pub char_limit: usize, + pub min_char_limit: usize, + pub max_char_limit: usize, + pub head_ratio: f32, + pub convert_base64_images: bool, + pub max_combined_inline_chars: usize, +} + +impl Default for WebExtractOptions { + fn default() -> Self { + Self { + char_limit: 15_000, + min_char_limit: 2_000, + max_char_limit: 500_000, + head_ratio: 0.75, + convert_base64_images: true, + max_combined_inline_chars: 100_000, + } + } +} + +/// Metadata-only reduction report for a web page. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct WebExtractReduction { + pub text: String, + pub body: String, + #[serde(default)] + pub recovery_footer: Option, + #[serde(default)] + pub ccr_token: Option, + pub source_host: Option, + pub source_url_hash: String, + #[serde(default)] + pub title: Option, + pub format: WebExtractFormat, + pub original_chars: usize, + pub inline_chars: usize, + pub head_chars: usize, + pub tail_chars: usize, + pub omitted_chars: usize, + pub truncated: bool, + pub full_text_retained: bool, + pub base64_images_replaced: usize, } impl ContentHint { @@ -424,6 +630,8 @@ pub enum CompressorKind { Html, /// ML (Python/ModernBERT) plain-text compressor. MlText, + /// Deterministic extractive plain-text compressor. + TextCrusher, /// Line-oriented head/tail fallback. Generic, /// No compressor fired — pass-through. @@ -441,6 +649,7 @@ impl CompressorKind { CompressorKind::Diff => "diff", CompressorKind::Html => "html", CompressorKind::MlText => "ml_text", + CompressorKind::TextCrusher => "textcrusher", CompressorKind::Generic => "generic", CompressorKind::None => "none", } @@ -602,13 +811,22 @@ impl Default for CompressOptions { } } -/// The result of the universal [`crate::compress_content`] -/// entry point: the compacted text (with any CCR footer already appended), plus -/// metadata for callers/stats. +/// The result of the universal [`crate::compress_content`] entry point. +/// +/// `text` preserves the original compatibility contract: it is the final +/// model-facing string with any CCR recovery footer already appended. Hosts that +/// apply their own downstream caps should use `body` and `recovery_footer` +/// instead, truncate only `body`, then reattach `recovery_footer` so the +/// recovery marker cannot be severed. #[derive(Debug, Clone)] pub struct CompressedOutput { - /// Final text to inline into context (includes the retrieval footer when lossy). + /// Final text to inline into context (includes the retrieval footer when present). pub text: String, + /// Compacted body without the recovery footer. + pub body: String, + /// Recovery footer to append after host-side truncation, if CCR retained the + /// original. + pub recovery_footer: Option, /// The detected content kind. pub content_kind: ContentKind, /// Which compressor fired (`None` ⇒ pass-through). @@ -630,7 +848,9 @@ impl CompressedOutput { pub fn passthrough(content: String, kind: ContentKind) -> Self { let len = content.len(); Self { - text: content, + text: content.clone(), + body: content, + recovery_footer: None, content_kind: kind, compressor: CompressorKind::None, lossy: false, diff --git a/src/vendor/rules/ci__github-actions.json b/src/vendor/rules/ci__github-actions.json new file mode 100644 index 0000000..8fcffb9 --- /dev/null +++ b/src/vendor/rules/ci__github-actions.json @@ -0,0 +1,41 @@ +{ + "id": "ci/github-actions", + "family": "test-results", + "description": "Compact GitHub Actions run logs from `gh run view --log` while preserving failing steps and annotations.", + "priority": 50, + "match": { + "toolNames": ["exec"], + "argv0": ["gh"], + "argvIncludes": [["run", "view"], ["--log"]] + }, + "transforms": { + "stripAnsi": true, + "dedupeAdjacent": true, + "trimEmptyEdges": true + }, + "filters": { + "keepPatterns": [ + "(?i)\\b(error|failed|failure|cancelled|timed out|process completed with exit code)\\b", + "(?i)^\\S+\\s+[^\\t]*\\b(run|build|test|lint)\\b" + ] + }, + "summarize": { + "head": 12, + "tail": 10 + }, + "failure": { + "preserveOnFailure": true, + "head": 18, + "tail": 18 + }, + "counters": [ + { + "name": "failed step", + "pattern": "(?i)\\bfailed\\b|process completed with exit code|##\\[error\\]|::error" + }, + { + "name": "error", + "pattern": "(?i)\\berror\\b|##\\[error\\]|::error" + } + ] +} diff --git a/tests/cli.rs b/tests/cli.rs new file mode 100644 index 0000000..9f859fa --- /dev/null +++ b/tests/cli.rs @@ -0,0 +1,574 @@ +use std::fs; +use std::io::Write; +use std::process::{Command, Stdio}; + +fn tinyjuice() -> Command { + Command::new(env!("CARGO_BIN_EXE_tinyjuice")) +} + +#[test] +fn reduce_compacts_stdin_with_command_metadata() { + let mut child = tinyjuice() + .args(["reduce", "--tool-name", "bash", "--command", "git status"]) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .spawn() + .expect("spawn tinyjuice reduce"); + + child + .stdin + .as_mut() + .expect("stdin") + .write_all(b"On branch main\n\nChanges not staged for commit:\n\tmodified: src/lib.rs\n") + .expect("write stdin"); + + let output = child.wait_with_output().expect("wait tinyjuice reduce"); + assert!(output.status.success(), "{output:#?}"); + assert_eq!( + String::from_utf8(output.stdout).expect("utf8 stdout"), + "Changes not staged:\nM: src/lib.rs" + ); +} + +#[test] +fn reduce_json_prints_response_json() { + let mut child = tinyjuice() + .args(["reduce-json"]) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .spawn() + .expect("spawn tinyjuice reduce-json"); + + child + .stdin + .as_mut() + .expect("stdin") + .write_all( + br#"{ + "toolName": "bash", + "argv": ["git", "status"], + "stdout": "On branch main\n\nChanges not staged for commit:\n\tmodified: src/lib.rs\n" + }"#, + ) + .expect("write stdin"); + + let output = child + .wait_with_output() + .expect("wait tinyjuice reduce-json"); + assert!(output.status.success(), "{output:#?}"); + let response: serde_json::Value = + serde_json::from_slice(&output.stdout).expect("response json"); + assert_eq!(response["inlineText"], "Changes not staged:\nM: src/lib.rs"); + assert_eq!(response["classification"]["matchedReducer"], "git/status"); +} + +#[test] +fn reduce_json_rejects_malformed_payload_with_structured_error() { + let mut child = tinyjuice() + .args(["reduce-json"]) + .stdin(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .expect("spawn tinyjuice reduce-json"); + + child + .stdin + .as_mut() + .expect("stdin") + .write_all(br#"{"toolName":"bash","stdout":"secret""#) + .expect("write stdin"); + + let output = child + .wait_with_output() + .expect("wait tinyjuice reduce-json"); + assert!(!output.status.success(), "{output:#?}"); + let error: serde_json::Value = serde_json::from_slice(&output.stderr).expect("error json"); + assert_eq!(error["code"], "invalid_json"); +} + +#[test] +fn verify_rules_and_fixtures_reports_counts() { + let output = tinyjuice() + .args(["verify", "--rules", "--fixtures"]) + .output() + .expect("run tinyjuice verify"); + + assert!(output.status.success(), "{output:#?}"); + let stdout = String::from_utf8(output.stdout).expect("utf8 stdout"); + assert!( + stdout.contains("rules: descriptors=101 valid=101 final=101"), + "{stdout}" + ); + assert!( + stdout.contains("fixtures: dir=tests/fixtures seen=5 passed=5"), + "{stdout}" + ); +} + +#[test] +fn discover_reports_fallback_families_without_raw_output() { + let mut child = tinyjuice() + .args(["discover"]) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .spawn() + .expect("spawn tinyjuice discover"); + + child + .stdin + .as_mut() + .expect("stdin") + .write_all( + br#"{"toolName":"bash","argv":["unknown_tool"],"stdout":"raw secret output"} +{"toolName":"bash","argv":["git","status"],"stdout":"On branch main"} +"#, + ) + .expect("write stdin"); + + let output = child.wait_with_output().expect("wait tinyjuice discover"); + assert!(output.status.success(), "{output:#?}"); + let response: serde_json::Value = + serde_json::from_slice(&output.stdout).expect("discovery json"); + assert_eq!(response["inputsSeen"], 2); + assert_eq!(response["fallbackOutputs"], 1); + assert_eq!(response["families"][0]["toolName"], "bash"); + assert_eq!(response["families"][0]["argv0"], "unknown_tool"); + let stdout = String::from_utf8(output.stdout).expect("utf8 stdout"); + assert!(!stdout.contains("raw secret output"), "{stdout}"); +} + +#[test] +fn wrap_runs_command_and_reduces_output() { + let mut script = tempfile::NamedTempFile::new().expect("temp script"); + writeln!( + script, + "i=1; while [ $i -le 80 ]; do echo line $i; i=$((i + 1)); done" + ) + .expect("write script"); + let script_path = script.path().to_string_lossy().into_owned(); + + let output = tinyjuice() + .args([ + "wrap", + "--max-inline-chars", + "120", + "--", + "sh", + &script_path, + ]) + .output() + .expect("run tinyjuice wrap"); + + assert!(output.status.success(), "{output:#?}"); + let stdout = String::from_utf8(output.stdout).expect("utf8 stdout"); + assert!(stdout.len() <= 120, "{stdout}"); + assert!(stdout.contains("omitted"), "{stdout}"); + assert!(!stdout.contains("line 40\nline 41"), "{stdout}"); +} + +#[test] +fn wrap_preserves_wrapped_exit_code() { + let output = tinyjuice() + .args(["wrap", "--", "sh", "-c", "printf 'line 1\\n'; exit 7"]) + .output() + .expect("run tinyjuice wrap failure"); + + assert_eq!(output.status.code(), Some(7), "{output:#?}"); + assert_eq!( + String::from_utf8(output.stdout).expect("utf8 stdout"), + "line 1\n" + ); +} + +#[test] +fn ls_lists_only_token_shaped_store_entries() { + let dir = tempfile::tempdir().expect("temp store"); + fs::write(dir.path().join("0123456789abcdef0123456789abcdef"), "one").expect("write token"); + fs::write(dir.path().join("not-a-token"), "ignored").expect("write ignored"); + + let output = tinyjuice() + .args(["ls", "--store-dir", &dir.path().to_string_lossy()]) + .output() + .expect("run tinyjuice ls"); + + assert!(output.status.success(), "{output:#?}"); + assert_eq!( + String::from_utf8(output.stdout).expect("utf8 stdout"), + "0123456789abcdef0123456789abcdef\n" + ); +} + +#[test] +fn cat_reads_token_from_explicit_store_dir() { + let dir = tempfile::tempdir().expect("temp store"); + let token = "0123456789abcdef0123456789abcdef"; + fs::write(dir.path().join(token), "alpha\nbeta\ngamma\n").expect("write token"); + + let output = tinyjuice() + .args(["cat", "--store-dir", &dir.path().to_string_lossy(), token]) + .output() + .expect("run tinyjuice cat"); + + assert!(output.status.success(), "{output:#?}"); + assert_eq!( + String::from_utf8(output.stdout).expect("utf8 stdout"), + "alpha\nbeta\ngamma\n" + ); +} + +#[test] +fn cat_reads_line_range_from_explicit_store_dir() { + let dir = tempfile::tempdir().expect("temp store"); + let token = "0123456789abcdef0123456789abcdef"; + fs::write(dir.path().join(token), "alpha\nbeta\ngamma\ndelta\n").expect("write token"); + + let output = tinyjuice() + .args([ + "cat", + "--store-dir", + &dir.path().to_string_lossy(), + "--lines", + "1:3", + token, + ]) + .output() + .expect("run tinyjuice cat lines"); + + assert!(output.status.success(), "{output:#?}"); + assert_eq!( + String::from_utf8(output.stdout).expect("utf8 stdout"), + "beta\ngamma" + ); +} + +#[test] +fn cat_reads_byte_range_from_explicit_store_dir() { + let dir = tempfile::tempdir().expect("temp store"); + let token = "0123456789abcdef0123456789abcdef"; + fs::write(dir.path().join(token), "alpha beta gamma").expect("write token"); + + let output = tinyjuice() + .args([ + "cat", + "--store-dir", + &dir.path().to_string_lossy(), + "--bytes", + "6:10", + token, + ]) + .output() + .expect("run tinyjuice cat bytes"); + + assert!(output.status.success(), "{output:#?}"); + assert_eq!( + String::from_utf8(output.stdout).expect("utf8 stdout"), + "beta" + ); +} + +#[test] +fn cat_rejects_multiple_range_flags() { + let dir = tempfile::tempdir().expect("temp store"); + let token = "0123456789abcdef0123456789abcdef"; + fs::write(dir.path().join(token), "alpha\nbeta\n").expect("write token"); + + let output = tinyjuice() + .args([ + "cat", + "--store-dir", + &dir.path().to_string_lossy(), + "--lines", + "0:1", + "--bytes", + "0:5", + token, + ]) + .output() + .expect("run tinyjuice cat conflicting ranges"); + + assert!(!output.status.success(), "{output:#?}"); + assert!( + String::from_utf8(output.stderr) + .expect("utf8 stderr") + .contains("cat accepts at most one range flag") + ); +} + +#[test] +fn stats_reports_metadata_for_explicit_store_dir() { + let dir = tempfile::tempdir().expect("temp store"); + fs::write(dir.path().join("0123456789abcdef0123456789abcdef"), "one").expect("write token"); + fs::write(dir.path().join("fedcba9876543210fedcba9876543210"), "three") + .expect("write second token"); + fs::write(dir.path().join("not-a-token"), "ignored raw content").expect("write ignored"); + fs::create_dir(dir.path().join("subdir")).expect("create ignored dir"); + + let output = tinyjuice() + .args(["stats", "--store-dir", &dir.path().to_string_lossy()]) + .output() + .expect("run tinyjuice stats"); + + assert!(output.status.success(), "{output:#?}"); + let response: serde_json::Value = serde_json::from_slice(&output.stdout).expect("stats json"); + assert_eq!(response["tokens"], 2); + assert_eq!(response["bytes"], 8); + assert_eq!(response["ignoredEntries"], 2); + assert_eq!(response["storeDir"], dir.path().to_string_lossy().as_ref()); + + let stdout = String::from_utf8(output.stdout).expect("utf8 stdout"); + assert!(!stdout.contains("ignored raw content"), "{stdout}"); +} + +#[test] +fn doctor_reports_health_checks_without_fixtures_by_default() { + let output = tinyjuice() + .args(["doctor"]) + .output() + .expect("run tinyjuice doctor"); + + assert!(output.status.success(), "{output:#?}"); + let response: serde_json::Value = serde_json::from_slice(&output.stdout).expect("doctor json"); + assert!( + matches!(response["status"].as_str(), Some("ok" | "warn")), + "{response:#?}" + ); + + let checks = response["checks"].as_array().expect("checks array"); + let rules = checks + .iter() + .find(|check| check["name"] == "rules") + .expect("rules check"); + assert!(matches!(rules["status"].as_str(), Some("ok" | "warn"))); + assert_eq!(rules["descriptors"], 101); + + let fixtures = checks + .iter() + .find(|check| check["name"] == "fixtures") + .expect("fixtures check"); + assert_eq!(fixtures["status"], "disabled"); +} + +#[test] +fn doctor_reports_broken_store_dir() { + let missing_dir = tempfile::tempdir() + .expect("temp parent") + .path() + .join("missing"); + let missing_dir = missing_dir.to_string_lossy().into_owned(); + + let output = tinyjuice() + .args(["doctor", "--store-dir", &missing_dir]) + .output() + .expect("run tinyjuice doctor"); + + assert!(!output.status.success(), "{output:#?}"); + let response: serde_json::Value = serde_json::from_slice(&output.stdout).expect("doctor json"); + assert_eq!(response["status"], "broken"); + let checks = response["checks"].as_array().expect("checks array"); + let store = checks + .iter() + .find(|check| check["name"] == "ccrDiskStore") + .expect("store check"); + assert_eq!(store["status"], "broken"); + assert_eq!(store["storeDir"], missing_dir); +} + +#[test] +fn doctor_codex_reports_disabled_without_managed_instructions() { + let codex_home = tempfile::tempdir().expect("temp codex home"); + + let output = tinyjuice() + .args([ + "doctor", + "codex", + "--codex-home", + &codex_home.path().to_string_lossy(), + ]) + .output() + .expect("run tinyjuice doctor codex"); + + assert!(output.status.success(), "{output:#?}"); + let response: serde_json::Value = serde_json::from_slice(&output.stdout).expect("doctor json"); + assert_eq!(response["status"], "disabled"); + assert_eq!(response["checks"][0]["name"], "codex"); + assert_eq!(response["checks"][0]["status"], "disabled"); + assert_eq!( + response["checks"][0]["repairCommand"], + "tinyjuice install codex" + ); +} + +#[test] +fn doctor_codex_reports_ok_for_existing_managed_command() { + let codex_home = tempfile::tempdir().expect("temp codex home"); + let instructions_dir = codex_home.path().join("instructions"); + fs::create_dir(&instructions_dir).expect("create instructions dir"); + let command = format!("{} reduce-json", env!("CARGO_BIN_EXE_tinyjuice")); + fs::write( + instructions_dir.join("tinyjuice.md"), + format!( + "user notes\n\nCommand: {command}\n\n" + ), + ) + .expect("write managed instructions"); + + let output = tinyjuice() + .args([ + "doctor", + "codex", + "--codex-home", + &codex_home.path().to_string_lossy(), + ]) + .output() + .expect("run tinyjuice doctor codex"); + + assert!(output.status.success(), "{output:#?}"); + let response: serde_json::Value = serde_json::from_slice(&output.stdout).expect("doctor json"); + assert_eq!(response["status"], "ok"); + assert_eq!(response["checks"][0]["status"], "ok"); + assert_eq!(response["checks"][0]["detectedCommand"], command); +} + +#[test] +fn doctor_codex_reports_broken_for_missing_managed_command() { + let codex_home = tempfile::tempdir().expect("temp codex home"); + let instructions_dir = codex_home.path().join("instructions"); + fs::create_dir(&instructions_dir).expect("create instructions dir"); + let command = codex_home + .path() + .join("missing-tinyjuice") + .to_string_lossy() + .into_owned(); + fs::write( + instructions_dir.join("tinyjuice.md"), + format!( + "\nCommand: {command} reduce-json\n\n" + ), + ) + .expect("write managed instructions"); + + let output = tinyjuice() + .args([ + "doctor", + "codex", + "--codex-home", + &codex_home.path().to_string_lossy(), + ]) + .output() + .expect("run tinyjuice doctor codex"); + + assert!(!output.status.success(), "{output:#?}"); + let response: serde_json::Value = serde_json::from_slice(&output.stdout).expect("doctor json"); + assert_eq!(response["status"], "broken"); + assert_eq!(response["checks"][0]["status"], "broken"); + assert_eq!( + response["checks"][0]["error"], + "managed command points at a missing executable" + ); +} + +#[test] +fn install_codex_writes_managed_block_and_is_idempotent() { + let codex_home = tempfile::tempdir().expect("temp codex home"); + let local = env!("CARGO_BIN_EXE_tinyjuice"); + + let first = tinyjuice() + .args([ + "install", + "codex", + "--codex-home", + &codex_home.path().to_string_lossy(), + "--local", + local, + ]) + .output() + .expect("run tinyjuice install codex"); + assert!(first.status.success(), "{first:#?}"); + let first_json: serde_json::Value = + serde_json::from_slice(&first.stdout).expect("install json"); + assert_eq!(first_json["status"], "ok"); + assert_eq!(first_json["changed"], true); + + let second = tinyjuice() + .args([ + "install", + "codex", + "--codex-home", + &codex_home.path().to_string_lossy(), + "--local", + local, + ]) + .output() + .expect("rerun tinyjuice install codex"); + assert!(second.status.success(), "{second:#?}"); + let second_json: serde_json::Value = + serde_json::from_slice(&second.stdout).expect("install json"); + assert_eq!(second_json["changed"], false); + + let instructions = + fs::read_to_string(codex_home.path().join("instructions/tinyjuice.md")).expect("read file"); + assert!(instructions.contains("")); + assert!(instructions.contains(&format!("Command: {local} reduce-json"))); +} + +#[test] +fn install_codex_preserves_unrelated_text_and_creates_backup() { + let codex_home = tempfile::tempdir().expect("temp codex home"); + let instructions_dir = codex_home.path().join("instructions"); + fs::create_dir(&instructions_dir).expect("create instructions dir"); + let path = instructions_dir.join("tinyjuice.md"); + fs::write(&path, "keep this note\n").expect("write existing instructions"); + + let output = tinyjuice() + .args([ + "install", + "codex", + "--codex-home", + &codex_home.path().to_string_lossy(), + "--local", + env!("CARGO_BIN_EXE_tinyjuice"), + ]) + .output() + .expect("run tinyjuice install codex"); + + assert!(output.status.success(), "{output:#?}"); + let updated = fs::read_to_string(&path).expect("read updated instructions"); + assert!(updated.contains("keep this note"), "{updated}"); + assert!(updated.contains("tinyjuice-managed:start"), "{updated}"); + assert_eq!( + fs::read_to_string(path.with_extension("md.bak")).expect("read backup"), + "keep this note\n" + ); +} + +#[test] +fn uninstall_codex_removes_only_managed_block() { + let codex_home = tempfile::tempdir().expect("temp codex home"); + let instructions_dir = codex_home.path().join("instructions"); + fs::create_dir(&instructions_dir).expect("create instructions dir"); + let path = instructions_dir.join("tinyjuice.md"); + fs::write( + &path, + "before\n\n\nCommand: /missing reduce-json\n\n\nafter\n", + ) + .expect("write managed instructions"); + + let output = tinyjuice() + .args([ + "uninstall", + "codex", + "--codex-home", + &codex_home.path().to_string_lossy(), + ]) + .output() + .expect("run tinyjuice uninstall codex"); + + assert!(output.status.success(), "{output:#?}"); + let response: serde_json::Value = + serde_json::from_slice(&output.stdout).expect("uninstall json"); + assert_eq!(response["changed"], true); + let updated = fs::read_to_string(&path).expect("read updated instructions"); + assert!(updated.contains("before"), "{updated}"); + assert!(updated.contains("after"), "{updated}"); + assert!(!updated.contains("tinyjuice-managed"), "{updated}"); +} diff --git a/tests/compressor_fixtures.rs b/tests/compressor_fixtures.rs new file mode 100644 index 0000000..c3e28c2 --- /dev/null +++ b/tests/compressor_fixtures.rs @@ -0,0 +1,599 @@ +//! Fixture-driven coverage for router-level content compressors. +//! +//! The command-rule fixtures in `tests/fixtures/` cover reducer rules. These +//! fixtures cover non-command content compressors through the store-injected +//! router path that hosts use. + +use serde::Deserialize; +use std::fmt::Write as _; +use std::path::PathBuf; +use tinyjuice::cache::{CcrStore, MemoryCcrStore}; +use tinyjuice::{ + CompressOptions, CompressorKind, ContentHint, ContentKind, compress_content_with_store_report, +}; + +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +struct CompressorFixture { + description: String, + kind: String, + #[serde(default)] + extension: Option, + #[serde(default)] + query: Option, + generator: FixtureGenerator, + expected: FixtureExpectations, +} + +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase", tag = "type")] +enum FixtureGenerator { + JsonRows { + #[serde(rename = "rows")] + rows: usize, + #[serde(default)] + #[serde(rename = "specialRow")] + special_row: Option, + #[serde(default)] + #[serde(rename = "specialNote")] + special_note: Option, + }, + JsonSparseRows { + #[serde(rename = "rows")] + rows: usize, + #[serde(rename = "sparseRow")] + sparse_row: usize, + }, + JsonErrorRows { + #[serde(rename = "rows")] + rows: usize, + #[serde(rename = "errorRow")] + error_row: usize, + }, + JsonNumericOutlierRows { + #[serde(rename = "rows")] + rows: usize, + #[serde(rename = "outlierRow")] + outlier_row: usize, + }, + JsonDuplicateClusterRows { + #[serde(rename = "rows")] + rows: usize, + #[serde(rename = "clusterStart")] + cluster_start: usize, + #[serde(rename = "clusterEnd")] + cluster_end: usize, + }, + JsonConstantRows { + #[serde(rename = "rows")] + rows: usize, + }, + JsonNestedRows { + #[serde(rename = "rows")] + rows: usize, + }, + JsonStringifiedRows { + #[serde(rename = "rows")] + rows: usize, + }, + JsonHeterogeneousRows { + #[serde(rename = "groups")] + groups: usize, + }, + JsonDiscriminatorRows { + #[serde(rename = "rows")] + rows: usize, + #[serde(rename = "specialRow")] + special_row: usize, + }, + JsonChangePointRows { + #[serde(rename = "rows")] + rows: usize, + #[serde(rename = "changeRow")] + change_row: usize, + }, + JsonInformationDenseRows { + #[serde(rename = "rows")] + rows: usize, + #[serde(rename = "denseRow")] + dense_row: usize, + }, + JsonNearDuplicateClusterRows { + #[serde(rename = "rows")] + rows: usize, + #[serde(rename = "clusterStart")] + cluster_start: usize, + #[serde(rename = "clusterEnd")] + cluster_end: usize, + }, + JsonSpreadRows { + #[serde(rename = "rows")] + rows: usize, + #[serde(rename = "spreadRow")] + spread_row: usize, + }, + JsonAdaptiveSaturationRows { + #[serde(rename = "rows")] + rows: usize, + }, + JsonEarlySaturationRows { + #[serde(rename = "rows")] + rows: usize, + }, + DiffLockfile { + #[serde(rename = "lines")] + lines: usize, + }, + DiffGeneratedBundle { + #[serde(rename = "lines")] + lines: usize, + }, + SearchResults { + #[serde(rename = "matchesPerFile")] + matches_per_file: usize, + }, + PlainStatus { + #[serde(rename = "before")] + before: usize, + #[serde(rename = "after")] + after: usize, + #[serde(rename = "errorLine")] + error_line: String, + }, +} + +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +struct FixtureExpectations { + compressor: String, + lossy: bool, + recovery: bool, + #[serde(default)] + contains: Vec, + #[serde(default)] + not_contains: Vec, +} + +#[tokio::test] +async fn content_compressor_fixtures_match_expectations() { + for (path, fixture) in load_compressor_fixtures() { + let input = fixture.generator.render(); + let hint = ContentHint { + extension: fixture.extension.clone(), + query: fixture.query.clone(), + explicit: Some(parse_content_kind(&fixture.kind, &path)), + ..ContentHint::default() + }; + let store = MemoryCcrStore::default(); + let (out, report) = + compress_content_with_store_report(&input, Some(hint), &fixture_options(), &store) + .await; + + let expected_compressor = parse_compressor_kind(&fixture.expected.compressor, &path); + assert_eq!( + out.compressor, + expected_compressor, + "{}: {} produced report {report:?}", + path.display(), + fixture.description + ); + assert_eq!( + out.lossy, + fixture.expected.lossy, + "{}: unexpected lossiness for {}", + path.display(), + fixture.description + ); + assert_eq!( + out.ccr_token.is_some(), + fixture.expected.recovery, + "{}: unexpected recovery token state for {}", + path.display(), + fixture.description + ); + + for needle in &fixture.expected.contains { + assert!( + out.text.contains(needle), + "{}: expected output to contain {needle:?}\n---\n{}\n---", + path.display(), + out.text + ); + } + for needle in &fixture.expected.not_contains { + assert!( + !out.text.contains(needle), + "{}: expected output not to contain {needle:?}\n---\n{}\n---", + path.display(), + out.text + ); + } + + if fixture.expected.recovery { + let token = out.ccr_token.as_deref().expect("checked above"); + assert_eq!( + store.get(token).as_deref(), + Some(input.as_str()), + "{}: recovery token did not retrieve original input", + path.display() + ); + } + } +} + +fn fixture_options() -> CompressOptions { + CompressOptions { + min_bytes_to_compress: 0, + ccr_min_tokens: 0, + ..CompressOptions::default() + } +} + +fn load_compressor_fixtures() -> Vec<(PathBuf, CompressorFixture)> { + let dir = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("tests/compressor_fixtures"); + let mut paths: Vec<_> = std::fs::read_dir(&dir) + .unwrap_or_else(|error| panic!("cannot read {}: {error}", dir.display())) + .filter_map(|entry| entry.ok().map(|entry| entry.path())) + .filter(|path| { + path.file_name() + .and_then(|name| name.to_str()) + .is_some_and(|name| name.ends_with(".fixture.json")) + }) + .collect(); + paths.sort(); + assert!( + !paths.is_empty(), + "no compressor fixtures found in {}", + dir.display() + ); + + paths + .into_iter() + .map(|path| { + let raw = std::fs::read_to_string(&path) + .unwrap_or_else(|error| panic!("cannot read {}: {error}", path.display())); + let fixture = serde_json::from_str(&raw) + .unwrap_or_else(|error| panic!("invalid fixture {}: {error}", path.display())); + (path, fixture) + }) + .collect() +} + +fn parse_content_kind(kind: &str, path: &std::path::Path) -> ContentKind { + match kind { + "json" => ContentKind::Json, + "diff" => ContentKind::Diff, + "search" => ContentKind::Search, + "plainText" => ContentKind::PlainText, + other => panic!("{}: unknown content kind {other}", path.display()), + } +} + +fn parse_compressor_kind(kind: &str, path: &std::path::Path) -> CompressorKind { + match kind { + "smartcrusher" => CompressorKind::SmartCrusher, + "diff" => CompressorKind::Diff, + "search" => CompressorKind::Search, + "textcrusher" => CompressorKind::TextCrusher, + other => panic!("{}: unknown compressor kind {other}", path.display()), + } +} + +impl FixtureGenerator { + fn render(&self) -> String { + match self { + FixtureGenerator::JsonRows { + rows, + special_row, + special_note, + } => { + let rendered_rows: Vec<_> = (0..*rows) + .map(|i| { + let note = if Some(i) == *special_row { + special_note.as_deref().unwrap_or("special row") + } else { + "ordinary row" + }; + format!( + r#"{{"id":{i},"name":"record {i}","status":"active","note":"{note}"}}"# + ) + }) + .collect(); + format!("[{}]", rendered_rows.join(",")) + } + FixtureGenerator::JsonSparseRows { rows, sparse_row } => { + let rendered_rows: Vec<_> = (0..*rows) + .map(|i| { + let extra = if i == *sparse_row { + r#","diagnostic":"rare sparse field""# + } else { + "" + }; + format!(r#"{{"id":{i},"name":"record {i}","status":"active"{extra}}}"#) + }) + .collect(); + format!("[{}]", rendered_rows.join(",")) + } + FixtureGenerator::JsonErrorRows { rows, error_row } => { + let rendered_rows: Vec<_> = (0..*rows) + .map(|i| { + let status = if i == *error_row { + "error: timeout" + } else { + "ok" + }; + format!( + r#"{{"id":{i},"name":"job {i}","status":"{status}","note":"detail {i}"}}"# + ) + }) + .collect(); + format!("[{}]", rendered_rows.join(",")) + } + FixtureGenerator::JsonNumericOutlierRows { rows, outlier_row } => { + let rendered_rows: Vec<_> = (0..*rows) + .map(|i| { + let latency = if i == *outlier_row { + 9999 + } else { + 10 + (i % 3) + }; + format!( + r#"{{"id":{i},"endpoint":"/api/{i}","latency_ms":{latency},"region":"us"}}"# + ) + }) + .collect(); + format!("[{}]", rendered_rows.join(",")) + } + FixtureGenerator::JsonDuplicateClusterRows { + rows, + cluster_start, + cluster_end, + } => { + let rendered_rows: Vec<_> = (0..*rows) + .map(|i| { + if (*cluster_start..=*cluster_end).contains(&i) { + r#"{"id":"retry-batch-17","status":"retry","message":"duplicate cluster payload"}"# + .to_owned() + } else { + format!( + r#"{{"id":"job-{i}","status":"ok","message":"ordinary payload {i}"}}"# + ) + } + }) + .collect(); + format!("[{}]", rendered_rows.join(",")) + } + FixtureGenerator::JsonConstantRows { rows } => { + let rendered_rows: Vec<_> = (0..*rows) + .map(|i| { + format!( + r#"{{"id":{i},"status":"active","region":"us-east","latency":{}}}"#, + 10 + i + ) + }) + .collect(); + format!("[{}]", rendered_rows.join(",")) + } + FixtureGenerator::JsonNestedRows { rows } => { + let rendered_rows: Vec<_> = (0..*rows) + .map(|i| { + format!( + r#"{{"id":{i},"user":{{"name":"user {i}","team":"core"}},"status":"active"}}"# + ) + }) + .collect(); + format!("[{}]", rendered_rows.join(",")) + } + FixtureGenerator::JsonStringifiedRows { rows } => { + let rendered_rows: Vec<_> = (0..*rows) + .map(|i| { + let metadata = serde_json::json!({ + "owner": format!("team-{i}"), + "flags": { "retry": i % 2 == 0 } + }) + .to_string() + .replace('"', "\\\""); + format!(r#"{{"id":{i},"metadata":"{metadata}","status":"active"}}"#) + }) + .collect(); + format!("[{}]", rendered_rows.join(",")) + } + FixtureGenerator::JsonHeterogeneousRows { groups } => { + let mut rendered_rows = Vec::new(); + for i in 0..*groups { + rendered_rows.push(format!( + r#"{{"event":"login","user_id":"user-{i}","ip":"10.0.0.{i}","success":true}}"# + )); + rendered_rows.push(format!( + r#"{{"event":"deploy","service":"api-{i}","version":"2026.{i}.0","region":"us-east"}}"# + )); + rendered_rows.push(format!( + r#"{{"event":"metric","name":"cpu-{i}","value":{},"unit":"pct"}}"#, + 40 + i + )); + } + format!("[{}]", rendered_rows.join(",")) + } + FixtureGenerator::JsonDiscriminatorRows { rows, special_row } => { + let rendered_rows: Vec<_> = (0..*rows) + .map(|i| { + let kind = if i == *special_row { + "audit" + } else if i % 2 == 0 { + "service" + } else { + "worker" + }; + let message = if i == *special_row { + "audit bucket unique payload" + } else { + "ordinary payload" + }; + format!( + r#"{{"id":{i},"kind":"{kind}","status":"active","message":"{message}"}}"# + ) + }) + .collect(); + format!("[{}]", rendered_rows.join(",")) + } + FixtureGenerator::JsonChangePointRows { rows, change_row } => { + let rendered_rows: Vec<_> = (0..*rows) + .map(|i| { + let phase = if i < *change_row { 10 } else { 30 }; + let message = if i == *change_row { + "phase transition payload".to_string() + } else { + format!("ordinary payload {i}") + }; + format!( + r#"{{"id":{i},"phase_score":{phase},"status":"active","message":"{message}"}}"# + ) + }) + .collect(); + format!("[{}]", rendered_rows.join(",")) + } + FixtureGenerator::JsonInformationDenseRows { rows, dense_row } => { + let rendered_rows: Vec<_> = (0..*rows) + .map(|i| { + let message = if i == *dense_row { + "trace alpha beta gamma delta epsilon zeta eta theta iota kappa lambda" + .to_string() + } else { + format!("ordinary payload {i}") + }; + format!(r#"{{"id":{i},"status":"active","message":"{message}"}}"#) + }) + .collect(); + format!("[{}]", rendered_rows.join(",")) + } + FixtureGenerator::JsonNearDuplicateClusterRows { + rows, + cluster_start, + cluster_end, + } => { + let rendered_rows: Vec<_> = (0..*rows) + .map(|i| { + let message = if (*cluster_start..=*cluster_end).contains(&i) { + "retry backlog service failed" + } else { + "routine healthy service stable" + }; + format!(r#"{{"id":"job-{i}","status":"active","message":"{message}"}}"#) + }) + .collect(); + format!("[{}]", rendered_rows.join(",")) + } + FixtureGenerator::JsonSpreadRows { rows, spread_row } => { + let rendered_rows: Vec<_> = (0..*rows) + .map(|i| { + let message = if i == *spread_row { + "deterministic spread anchor payload".to_string() + } else { + format!("ordinary payload {i}") + }; + format!( + r#"{{"id":{i},"name":"record {i}","status":"active","message":"{message}"}}"# + ) + }) + .collect(); + format!("[{}]", rendered_rows.join(",")) + } + FixtureGenerator::JsonAdaptiveSaturationRows { rows } => { + let rendered_rows: Vec<_> = (0..*rows) + .map(|i| { + let word = alpha_word(i); + format!( + r#"{{"id":{i},"status":"active","message":"topic {word} marker alpha beta"}}"# + ) + }) + .collect(); + format!("[{}]", rendered_rows.join(",")) + } + FixtureGenerator::JsonEarlySaturationRows { rows } => { + let rendered_rows: Vec<_> = (0..*rows) + .map(|i| { + let message = if i < (*rows / 2) { + format!("topic {} marker alpha beta", alpha_word(i)) + } else { + "topic repeated marker alpha beta".to_string() + }; + format!(r#"{{"id":"job-{i}","status":"active","message":"{message}"}}"#) + }) + .collect(); + format!("[{}]", rendered_rows.join(",")) + } + FixtureGenerator::DiffLockfile { lines } => { + let mut out = + String::from("diff --git a/Cargo.lock b/Cargo.lock\n@@ -1,80 +1,80 @@\n"); + for i in 0..*lines { + let _ = writeln!(out, "+ new dep entry {i}"); + } + for i in 0..*lines { + let _ = writeln!(out, "- old dep entry {i}"); + } + out + } + FixtureGenerator::DiffGeneratedBundle { lines } => { + let mut out = String::from( + "diff --git a/dist/app.min.js b/dist/app.min.js\n@@ -1,80 +1,80 @@\n", + ); + for i in 0..*lines { + let _ = writeln!(out, "+ minified chunk payload {i}"); + } + for i in 0..*lines { + let _ = writeln!(out, "- previous minified payload {i}"); + } + out + } + FixtureGenerator::SearchResults { matches_per_file } => { + let mut out = format!("{} match(es); scanned 2 file(s)\n", matches_per_file * 2); + for i in 0..*matches_per_file { + let body = if i == 7 { + "let needle_value = compute_long_name_7();".to_owned() + } else { + format!("let value_{i} = compute_long_name_{i}();") + }; + let _ = writeln!(out, "src/a.rs:{i}:{body}"); + } + for i in 0..*matches_per_file { + let _ = writeln!(out, "src/b.rs:{i}:fn helper_function_number_{i}() {{}}"); + } + out + } + FixtureGenerator::PlainStatus { + before, + after, + error_line, + } => { + let mut out = String::new(); + for i in 0..*before { + let _ = writeln!( + out, + "ordinary deployment progress line {i} with routine status information.\n" + ); + } + let _ = writeln!(out, "{error_line}\n"); + for i in 0..*after { + let _ = writeln!( + out, + "ordinary deployment progress line {} with routine status information.\n", + before + i + ); + } + out + } + } + } +} + +fn alpha_word(mut n: usize) -> String { + let mut out = String::from("word"); + loop { + out.push((b'a' + (n % 26) as u8) as char); + n /= 26; + if n == 0 { + break; + } + } + out +} diff --git a/tests/compressor_fixtures/diff_generated_bundle.fixture.json b/tests/compressor_fixtures/diff_generated_bundle.fixture.json new file mode 100644 index 0000000..3d190a4 --- /dev/null +++ b/tests/compressor_fixtures/diff_generated_bundle.fixture.json @@ -0,0 +1,21 @@ +{ + "description": "DiffNoise collapses generated bundle body with a clear reason and retained recovery", + "kind": "diff", + "extension": "diff", + "generator": { + "type": "diffGeneratedBundle", + "lines": 80 + }, + "expected": { + "compressor": "diff", + "lossy": true, + "recovery": true, + "contains": [ + "reason=generated_bundle", + "tinyjuice_retrieve" + ], + "notContains": [ + "minified chunk payload 79" + ] + } +} diff --git a/tests/compressor_fixtures/diff_lockfile.fixture.json b/tests/compressor_fixtures/diff_lockfile.fixture.json new file mode 100644 index 0000000..816f624 --- /dev/null +++ b/tests/compressor_fixtures/diff_lockfile.fixture.json @@ -0,0 +1,21 @@ +{ + "description": "DiffNoise collapses generated lockfile body with a clear reason and retained recovery", + "kind": "diff", + "extension": "diff", + "generator": { + "type": "diffLockfile", + "lines": 80 + }, + "expected": { + "compressor": "diff", + "lossy": true, + "recovery": true, + "contains": [ + "reason=lockfile", + "tinyjuice_retrieve" + ], + "notContains": [ + "new dep entry 79" + ] + } +} diff --git a/tests/compressor_fixtures/json_adaptive_saturation_anchor.fixture.json b/tests/compressor_fixtures/json_adaptive_saturation_anchor.fixture.json new file mode 100644 index 0000000..cea8d84 --- /dev/null +++ b/tests/compressor_fixtures/json_adaptive_saturation_anchor.fixture.json @@ -0,0 +1,18 @@ +{ + "description": "SmartCrusher row dropping adds adaptive spread anchors when semantic bigrams do not saturate early", + "kind": "json", + "extension": "json", + "generator": { + "type": "jsonAdaptiveSaturationRows", + "rows": 140 + }, + "expected": { + "compressor": "smartcrusher", + "lossy": true, + "recovery": true, + "contains": [ + "topic wordee marker alpha beta", + "tinyjuice_retrieve" + ] + } +} diff --git a/tests/compressor_fixtures/json_constant_columns.fixture.json b/tests/compressor_fixtures/json_constant_columns.fixture.json new file mode 100644 index 0000000..9074600 --- /dev/null +++ b/tests/compressor_fixtures/json_constant_columns.fixture.json @@ -0,0 +1,22 @@ +{ + "description": "SmartCrusher table rendering hoists full-present constant columns", + "kind": "json", + "extension": "json", + "generator": { + "type": "jsonConstantRows", + "rows": 20 + }, + "expected": { + "compressor": "smartcrusher", + "lossy": false, + "recovery": false, + "contains": [ + "constants: region=us-east | status=active", + "id | latency", + "7 | 17" + ], + "notContains": [ + "id | latency | region" + ] + } +} diff --git a/tests/compressor_fixtures/json_discriminator_anchor.fixture.json b/tests/compressor_fixtures/json_discriminator_anchor.fixture.json new file mode 100644 index 0000000..d17d34a --- /dev/null +++ b/tests/compressor_fixtures/json_discriminator_anchor.fixture.json @@ -0,0 +1,23 @@ +{ + "description": "SmartCrusher row dropping preserves discriminator bucket representatives", + "kind": "json", + "extension": "json", + "generator": { + "type": "jsonDiscriminatorRows", + "rows": 140, + "specialRow": 72 + }, + "expected": { + "compressor": "smartcrusher", + "lossy": true, + "recovery": true, + "contains": [ + "audit", + "audit bucket unique payload", + "tinyjuice_retrieve" + ], + "notContains": [ + "ordinary payload 70" + ] + } +} diff --git a/tests/compressor_fixtures/json_duplicate_cluster_anchor.fixture.json b/tests/compressor_fixtures/json_duplicate_cluster_anchor.fixture.json new file mode 100644 index 0000000..6311f2b --- /dev/null +++ b/tests/compressor_fixtures/json_duplicate_cluster_anchor.fixture.json @@ -0,0 +1,24 @@ +{ + "description": "SmartCrusher row dropping preserves duplicate-cluster representatives", + "kind": "json", + "extension": "json", + "generator": { + "type": "jsonDuplicateClusterRows", + "rows": 140, + "clusterStart": 72, + "clusterEnd": 76 + }, + "expected": { + "compressor": "smartcrusher", + "lossy": true, + "recovery": true, + "contains": [ + "retry-batch-17", + "duplicate cluster payload", + "tinyjuice_retrieve" + ], + "notContains": [ + "ordinary payload 70" + ] + } +} diff --git a/tests/compressor_fixtures/json_early_saturation_anchor.fixture.json b/tests/compressor_fixtures/json_early_saturation_anchor.fixture.json new file mode 100644 index 0000000..e2de98e --- /dev/null +++ b/tests/compressor_fixtures/json_early_saturation_anchor.fixture.json @@ -0,0 +1,19 @@ +{ + "description": "SmartCrusher row dropping keeps base spread anchors when semantic bigrams saturate early", + "kind": "json", + "extension": "json", + "generator": { + "type": "jsonEarlySaturationRows", + "rows": 140 + }, + "expected": { + "compressor": "smartcrusher", + "lossy": true, + "recovery": true, + "contains": [ + "job-94", + "topic repeated marker alpha beta", + "tinyjuice_retrieve" + ] + } +} diff --git a/tests/compressor_fixtures/json_error_anchor.fixture.json b/tests/compressor_fixtures/json_error_anchor.fixture.json new file mode 100644 index 0000000..9e5b917 --- /dev/null +++ b/tests/compressor_fixtures/json_error_anchor.fixture.json @@ -0,0 +1,23 @@ +{ + "description": "SmartCrusher row dropping preserves buried error rows", + "kind": "json", + "extension": "json", + "generator": { + "type": "jsonErrorRows", + "rows": 120, + "errorRow": 75 + }, + "expected": { + "compressor": "smartcrusher", + "lossy": true, + "recovery": true, + "contains": [ + "job 75", + "error: timeout", + "tinyjuice_retrieve" + ], + "notContains": [ + "job 70" + ] + } +} diff --git a/tests/compressor_fixtures/json_heterogeneous_buckets.fixture.json b/tests/compressor_fixtures/json_heterogeneous_buckets.fixture.json new file mode 100644 index 0000000..f99f4eb --- /dev/null +++ b/tests/compressor_fixtures/json_heterogeneous_buckets.fixture.json @@ -0,0 +1,27 @@ +{ + "description": "SmartCrusher renders heterogeneous object arrays as per-shape bucket tables", + "kind": "json", + "extension": "json", + "generator": { + "type": "jsonHeterogeneousRows", + "groups": 12 + }, + "expected": { + "compressor": "smartcrusher", + "lossy": false, + "recovery": false, + "contains": [ + "[json bucketed tables:", + "[bucket 1:", + "[bucket 2:", + "[bucket 3:", + "constants: event=login | success=true", + "constants: event=deploy | region=us-east", + "constants: event=metric | unit=pct", + "__row | ip | user_id", + "__row | service | version", + "__row | name | value", + "31 | api-10 | 2026.10.0" + ] + } +} diff --git a/tests/compressor_fixtures/json_information_dense_anchor.fixture.json b/tests/compressor_fixtures/json_information_dense_anchor.fixture.json new file mode 100644 index 0000000..d55dffc --- /dev/null +++ b/tests/compressor_fixtures/json_information_dense_anchor.fixture.json @@ -0,0 +1,22 @@ +{ + "description": "SmartCrusher row dropping preserves information-dense rows", + "kind": "json", + "extension": "json", + "generator": { + "type": "jsonInformationDenseRows", + "rows": 140, + "denseRow": 72 + }, + "expected": { + "compressor": "smartcrusher", + "lossy": true, + "recovery": true, + "contains": [ + "trace alpha beta gamma", + "tinyjuice_retrieve" + ], + "notContains": [ + "ordinary payload 70" + ] + } +} diff --git a/tests/compressor_fixtures/json_latest_query_anchor.fixture.json b/tests/compressor_fixtures/json_latest_query_anchor.fixture.json new file mode 100644 index 0000000..5288b5f --- /dev/null +++ b/tests/compressor_fixtures/json_latest_query_anchor.fixture.json @@ -0,0 +1,24 @@ +{ + "description": "SmartCrusher row dropping extends tail anchors for latest/recent queries", + "kind": "json", + "extension": "json", + "query": "latest records", + "generator": { + "type": "jsonRows", + "rows": 140, + "specialRow": 124, + "specialNote": "latest positional context" + }, + "expected": { + "compressor": "smartcrusher", + "lossy": true, + "recovery": true, + "contains": [ + "latest positional context", + "tinyjuice_retrieve" + ], + "notContains": [ + "record 70" + ] + } +} diff --git a/tests/compressor_fixtures/json_near_duplicate_cluster_anchor.fixture.json b/tests/compressor_fixtures/json_near_duplicate_cluster_anchor.fixture.json new file mode 100644 index 0000000..04e8f46 --- /dev/null +++ b/tests/compressor_fixtures/json_near_duplicate_cluster_anchor.fixture.json @@ -0,0 +1,21 @@ +{ + "description": "SmartCrusher row dropping preserves near-duplicate cluster representatives", + "kind": "json", + "extension": "json", + "generator": { + "type": "jsonNearDuplicateClusterRows", + "rows": 140, + "clusterStart": 72, + "clusterEnd": 76 + }, + "expected": { + "compressor": "smartcrusher", + "lossy": true, + "recovery": true, + "contains": [ + "job-72", + "retry backlog service failed", + "tinyjuice_retrieve" + ] + } +} diff --git a/tests/compressor_fixtures/json_nested_columns.fixture.json b/tests/compressor_fixtures/json_nested_columns.fixture.json new file mode 100644 index 0000000..633424b --- /dev/null +++ b/tests/compressor_fixtures/json_nested_columns.fixture.json @@ -0,0 +1,19 @@ +{ + "description": "SmartCrusher table rendering flattens nested object columns", + "kind": "json", + "extension": "json", + "generator": { + "type": "jsonNestedRows", + "rows": 20 + }, + "expected": { + "compressor": "smartcrusher", + "lossy": false, + "recovery": false, + "contains": [ + "user.name", + "user.team", + "user 7" + ] + } +} diff --git a/tests/compressor_fixtures/json_numeric_change_point_anchor.fixture.json b/tests/compressor_fixtures/json_numeric_change_point_anchor.fixture.json new file mode 100644 index 0000000..39e09a4 --- /dev/null +++ b/tests/compressor_fixtures/json_numeric_change_point_anchor.fixture.json @@ -0,0 +1,22 @@ +{ + "description": "SmartCrusher row dropping preserves numeric change-point rows", + "kind": "json", + "extension": "json", + "generator": { + "type": "jsonChangePointRows", + "rows": 140, + "changeRow": 72 + }, + "expected": { + "compressor": "smartcrusher", + "lossy": true, + "recovery": true, + "contains": [ + "phase transition payload", + "tinyjuice_retrieve" + ], + "notContains": [ + "ordinary payload 70" + ] + } +} diff --git a/tests/compressor_fixtures/json_numeric_outlier_anchor.fixture.json b/tests/compressor_fixtures/json_numeric_outlier_anchor.fixture.json new file mode 100644 index 0000000..3d02ef6 --- /dev/null +++ b/tests/compressor_fixtures/json_numeric_outlier_anchor.fixture.json @@ -0,0 +1,23 @@ +{ + "description": "SmartCrusher row dropping preserves numeric outlier rows", + "kind": "json", + "extension": "json", + "generator": { + "type": "jsonNumericOutlierRows", + "rows": 120, + "outlierRow": 88 + }, + "expected": { + "compressor": "smartcrusher", + "lossy": true, + "recovery": true, + "contains": [ + "9999", + "/api/88", + "tinyjuice_retrieve" + ], + "notContains": [ + "/api/70" + ] + } +} diff --git a/tests/compressor_fixtures/json_oldest_query_anchor.fixture.json b/tests/compressor_fixtures/json_oldest_query_anchor.fixture.json new file mode 100644 index 0000000..00d7fd5 --- /dev/null +++ b/tests/compressor_fixtures/json_oldest_query_anchor.fixture.json @@ -0,0 +1,24 @@ +{ + "description": "SmartCrusher row dropping extends head anchors for oldest/initial queries", + "kind": "json", + "extension": "json", + "query": "oldest records", + "generator": { + "type": "jsonRows", + "rows": 140, + "specialRow": 24, + "specialNote": "oldest positional context" + }, + "expected": { + "compressor": "smartcrusher", + "lossy": true, + "recovery": true, + "contains": [ + "oldest positional context", + "tinyjuice_retrieve" + ], + "notContains": [ + "record 70" + ] + } +} diff --git a/tests/compressor_fixtures/json_query_anchor.fixture.json b/tests/compressor_fixtures/json_query_anchor.fixture.json new file mode 100644 index 0000000..80e7655 --- /dev/null +++ b/tests/compressor_fixtures/json_query_anchor.fixture.json @@ -0,0 +1,24 @@ +{ + "description": "SmartCrusher row dropping keeps query-relevant rows and retains the original", + "kind": "json", + "extension": "json", + "query": "special needle", + "generator": { + "type": "jsonRows", + "rows": 140, + "specialRow": 71, + "specialNote": "contains special needle token" + }, + "expected": { + "compressor": "smartcrusher", + "lossy": true, + "recovery": true, + "contains": [ + "special needle token", + "tinyjuice_retrieve" + ], + "notContains": [ + "\"id\": 70" + ] + } +} diff --git a/tests/compressor_fixtures/json_sparse_anchor.fixture.json b/tests/compressor_fixtures/json_sparse_anchor.fixture.json new file mode 100644 index 0000000..d21ed0e --- /dev/null +++ b/tests/compressor_fixtures/json_sparse_anchor.fixture.json @@ -0,0 +1,22 @@ +{ + "description": "SmartCrusher row dropping preserves sparse structural rows", + "kind": "json", + "extension": "json", + "generator": { + "type": "jsonSparseRows", + "rows": 140, + "sparseRow": 72 + }, + "expected": { + "compressor": "smartcrusher", + "lossy": true, + "recovery": true, + "contains": [ + "rare sparse field", + "tinyjuice_retrieve" + ], + "notContains": [ + "record 70" + ] + } +} diff --git a/tests/compressor_fixtures/json_spread_anchor.fixture.json b/tests/compressor_fixtures/json_spread_anchor.fixture.json new file mode 100644 index 0000000..f721f50 --- /dev/null +++ b/tests/compressor_fixtures/json_spread_anchor.fixture.json @@ -0,0 +1,22 @@ +{ + "description": "SmartCrusher row dropping keeps deterministic spread anchors", + "kind": "json", + "extension": "json", + "generator": { + "type": "jsonSpreadRows", + "rows": 140, + "spreadRow": 94 + }, + "expected": { + "compressor": "smartcrusher", + "lossy": true, + "recovery": true, + "contains": [ + "deterministic spread anchor payload", + "tinyjuice_retrieve" + ], + "notContains": [ + "ordinary payload 70" + ] + } +} diff --git a/tests/compressor_fixtures/json_stringified_columns.fixture.json b/tests/compressor_fixtures/json_stringified_columns.fixture.json new file mode 100644 index 0000000..9049c92 --- /dev/null +++ b/tests/compressor_fixtures/json_stringified_columns.fixture.json @@ -0,0 +1,19 @@ +{ + "description": "SmartCrusher table rendering flattens small stringified JSON object columns", + "kind": "json", + "extension": "json", + "generator": { + "type": "jsonStringifiedRows", + "rows": 20 + }, + "expected": { + "compressor": "smartcrusher", + "lossy": false, + "recovery": false, + "contains": [ + "metadata.owner", + "metadata.flags.retry", + "team-7" + ] + } +} diff --git a/tests/compressor_fixtures/search_ranked.fixture.json b/tests/compressor_fixtures/search_ranked.fixture.json new file mode 100644 index 0000000..ee02d70 --- /dev/null +++ b/tests/compressor_fixtures/search_ranked.fixture.json @@ -0,0 +1,22 @@ +{ + "description": "Search thinning preserves query-ranked match, omitted counts, and recovery", + "kind": "search", + "query": "needle_value", + "generator": { + "type": "searchResults", + "matchesPerFile": 60 + }, + "expected": { + "compressor": "search", + "lossy": true, + "recovery": true, + "contains": [ + "needle_value", + "search omitted", + "tinyjuice_retrieve" + ], + "notContains": [ + "helper_function_number_59" + ] + } +} diff --git a/tests/compressor_fixtures/textcrusher_signal.fixture.json b/tests/compressor_fixtures/textcrusher_signal.fixture.json new file mode 100644 index 0000000..0f71950 --- /dev/null +++ b/tests/compressor_fixtures/textcrusher_signal.fixture.json @@ -0,0 +1,23 @@ +{ + "description": "TextCrusher keeps query-relevant error text verbatim and retains recovery", + "kind": "plainText", + "query": "sync.worker.v2 REQUEST_ID", + "generator": { + "type": "plainStatus", + "before": 40, + "after": 40, + "errorLine": "ERROR sync.worker.v2 failed for REQUEST_ID 9F42 after retry 17." + }, + "expected": { + "compressor": "textcrusher", + "lossy": true, + "recovery": true, + "contains": [ + "ERROR sync.worker.v2 failed for REQUEST_ID 9F42", + "tinyjuice_retrieve" + ], + "notContains": [ + "ordinary deployment progress line 3 with" + ] + } +} diff --git a/tests/diff_noise_policy_fixtures.rs b/tests/diff_noise_policy_fixtures.rs new file mode 100644 index 0000000..1d363eb --- /dev/null +++ b/tests/diff_noise_policy_fixtures.rs @@ -0,0 +1,183 @@ +//! Fixture coverage for host-tuned DiffNoise options. + +use serde::Deserialize; +use std::fmt::Write as _; +use std::path::PathBuf; +use tinyjuice::cache::{CcrStore, MemoryCcrStore}; +use tinyjuice::compressors::diff::{DiffNoiseOptions, DiffNoiseTransform}; +use tinyjuice::{ContentKind, OffloadTransform, PipelineInput}; + +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +struct DiffNoisePolicyFixture { + description: String, + generator: FixtureGenerator, + options: FixtureOptions, + expected: FixtureExpectations, +} + +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase", tag = "type")] +enum FixtureGenerator { + ConfiguredPath { + #[serde(rename = "lines")] + lines: usize, + }, + WhitespaceOnly { + #[serde(rename = "context")] + context: usize, + }, + Semantic { + #[serde(rename = "context")] + context: usize, + }, +} + +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +struct FixtureOptions { + #[serde(default)] + noisy_path_substrings: Vec, + #[serde(default)] + drop_whitespace_only_hunks: bool, +} + +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +struct FixtureExpectations { + recovery: bool, + #[serde(default)] + contains: Vec, + #[serde(default)] + not_contains: Vec, +} + +#[test] +fn diff_noise_policy_fixtures_match_expectations() { + for (path, fixture) in load_diff_noise_policy_fixtures() { + let input = fixture.generator.render(); + let options = DiffNoiseOptions { + noisy_path_substrings: fixture.options.noisy_path_substrings, + drop_whitespace_only_hunks: fixture.options.drop_whitespace_only_hunks, + }; + let transform = DiffNoiseTransform::new(options); + let pipeline_input = PipelineInput { + content: &input, + original_content: &input, + content_kind: ContentKind::Diff, + original_bytes: input.len(), + }; + let store = MemoryCcrStore::default(); + let out = transform.apply(&pipeline_input, &store).unwrap_or_else(|| { + panic!( + "{}: fixture did not produce DiffNoise output for {}", + path.display(), + fixture.description + ) + }); + + for needle in &fixture.expected.contains { + assert!( + out.text().contains(needle), + "{}: expected output to contain {needle:?} for {}\n---\n{}\n---", + path.display(), + fixture.description, + out.text() + ); + } + for needle in &fixture.expected.not_contains { + assert!( + !out.text().contains(needle), + "{}: expected output not to contain {needle:?} for {}\n---\n{}\n---", + path.display(), + fixture.description, + out.text() + ); + } + + if fixture.expected.recovery { + assert_eq!( + store.get(out.token()).as_deref(), + Some(input.as_str()), + "{}: recovery token did not retrieve original input for {}", + path.display(), + fixture.description + ); + } + } +} + +fn load_diff_noise_policy_fixtures() -> Vec<(PathBuf, DiffNoisePolicyFixture)> { + let dir = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("tests/diff_noise_policy_fixtures"); + let mut paths: Vec<_> = std::fs::read_dir(&dir) + .unwrap_or_else(|error| panic!("cannot read {}: {error}", dir.display())) + .filter_map(|entry| entry.ok().map(|entry| entry.path())) + .filter(|path| { + path.file_name() + .and_then(|name| name.to_str()) + .is_some_and(|name| name.ends_with(".fixture.json")) + }) + .collect(); + paths.sort(); + assert!( + !paths.is_empty(), + "no DiffNoise policy fixtures found in {}", + dir.display() + ); + + paths + .into_iter() + .map(|path| { + let raw = std::fs::read_to_string(&path) + .unwrap_or_else(|error| panic!("cannot read {}: {error}", path.display())); + let fixture = serde_json::from_str(&raw) + .unwrap_or_else(|error| panic!("invalid fixture {}: {error}", path.display())); + (path, fixture) + }) + .collect() +} + +impl FixtureGenerator { + fn render(&self) -> String { + match self { + FixtureGenerator::ConfiguredPath { lines } => { + let mut out = String::from( + "diff --git a/fixtures/snapshot.txt b/fixtures/snapshot.txt\n@@ -1,80 +1,80 @@\n", + ); + for i in 0..*lines { + let _ = writeln!(out, "+ snapshot chunk payload {i}"); + } + for i in 0..*lines { + let _ = writeln!(out, "- previous snapshot payload {i}"); + } + out + } + FixtureGenerator::WhitespaceOnly { context } => { + let mut out = + String::from("diff --git a/src/lib.rs b/src/lib.rs\n@@ -1,60 +1,60 @@\n"); + for i in 0..*context { + let _ = writeln!(out, " context before {i}"); + } + let _ = writeln!(out, "-fn main(){{println!(\"hi\");}}"); + let _ = writeln!(out, "+fn main() {{ println!(\"hi\"); }}"); + for i in 0..*context { + let _ = writeln!(out, " context after {i}"); + } + out + } + FixtureGenerator::Semantic { context } => { + let mut out = + String::from("diff --git a/src/lib.rs b/src/lib.rs\n@@ -1,40 +1,40 @@\n"); + for i in 0..*context { + let _ = writeln!(out, " context before {i}"); + } + let _ = writeln!(out, "-fn answer() -> i32 {{ 41 }}"); + let _ = writeln!(out, "+fn answer() -> i32 {{ 42 }}"); + for i in 0..*context { + let _ = writeln!(out, " context after {i}"); + } + out + } + } + } +} diff --git a/tests/diff_noise_policy_fixtures/configured_path.fixture.json b/tests/diff_noise_policy_fixtures/configured_path.fixture.json new file mode 100644 index 0000000..56b687d --- /dev/null +++ b/tests/diff_noise_policy_fixtures/configured_path.fixture.json @@ -0,0 +1,21 @@ +{ + "description": "DiffNoise configured noisy path patterns drop matching hunks with a configured reason", + "generator": { + "type": "configuredPath", + "lines": 40 + }, + "options": { + "noisyPathSubstrings": [ + "snapshot.txt" + ] + }, + "expected": { + "recovery": true, + "contains": [ + "reason=configured_noisy_path" + ], + "notContains": [ + "snapshot chunk payload 39" + ] + } +} diff --git a/tests/diff_noise_policy_fixtures/semantic_hunk.fixture.json b/tests/diff_noise_policy_fixtures/semantic_hunk.fixture.json new file mode 100644 index 0000000..b35aa16 --- /dev/null +++ b/tests/diff_noise_policy_fixtures/semantic_hunk.fixture.json @@ -0,0 +1,21 @@ +{ + "description": "DiffNoise whitespace-only mode preserves semantic hunks", + "generator": { + "type": "semantic", + "context": 50 + }, + "options": { + "dropWhitespaceOnlyHunks": true + }, + "expected": { + "recovery": true, + "contains": [ + "-fn answer() -> i32 { 41 }", + "+fn answer() -> i32 { 42 }" + ], + "notContains": [ + "reason=whitespace_only", + "diff-noise hunk omitted" + ] + } +} diff --git a/tests/diff_noise_policy_fixtures/whitespace_only.fixture.json b/tests/diff_noise_policy_fixtures/whitespace_only.fixture.json new file mode 100644 index 0000000..fd85539 --- /dev/null +++ b/tests/diff_noise_policy_fixtures/whitespace_only.fixture.json @@ -0,0 +1,20 @@ +{ + "description": "DiffNoise whitespace-only mode drops paired changes that normalize equal", + "generator": { + "type": "whitespaceOnly", + "context": 50 + }, + "options": { + "dropWhitespaceOnlyHunks": true + }, + "expected": { + "recovery": true, + "contains": [ + "reason=whitespace_only" + ], + "notContains": [ + "-fn main()", + "+fn main()" + ] + } +} diff --git a/tests/e2e_tool_output.rs b/tests/e2e_tool_output.rs index c0fefec..2dbeabd 100644 --- a/tests/e2e_tool_output.rs +++ b/tests/e2e_tool_output.rs @@ -3,12 +3,13 @@ //! These exercise the same path OpenHuman's `ToolOutputMiddleware` uses: //! `compact_tool_output_with_policy` → router → compressor → CCR offload → //! recovery footer → `cache::retrieve` roundtrip. They pin the plan-level -//! acceptance criteria: `off` and recovery-tool output are exact passthrough, -//! `light` declines lossy compaction, `full` compacts recoverably. +//! acceptance criteria: `off`, recovery-tool output, and exact file reads are +//! passthrough; `light` declines lossy compaction; `full` compacts recoverably. mod common; use serde_json::json; +use std::fmt::Write as _; use tinyjuice::cache; use tinyjuice::tool_integration::compact_tool_output_with_policy; use tinyjuice::types::AgentTokenjuiceCompression; @@ -31,6 +32,72 @@ fn big_json_rows() -> String { serde_json::to_string_pretty(&rows).unwrap() } +fn lockfile_diff() -> String { + let mut s = String::from("diff --git a/Cargo.lock b/Cargo.lock\n@@ -1,80 +1,80 @@\n"); + for i in 0..80 { + let _ = writeln!(s, "+ new dep entry {i}"); + } + for i in 0..80 { + let _ = writeln!(s, "- old dep entry {i}"); + } + s +} + +fn search_results() -> String { + let mut s = String::from("120 match(es); scanned 2 file(s)\n"); + for i in 0..60 { + let body = if i == 7 { + "let needle_value = compute_long_name_7();".to_owned() + } else { + format!("let value_{i} = compute_long_name_{i}();") + }; + let _ = writeln!(s, "src/a.rs:{i}:{body}"); + } + for i in 0..60 { + let _ = writeln!(s, "src/b.rs:{i}:fn helper_function_number_{i}() {{}}"); + } + s +} + +fn status_report_text() -> String { + let mut s = String::new(); + for i in 0..40 { + let _ = writeln!( + s, + "ordinary deployment progress line {i} with routine status information.\n" + ); + } + s.push_str("ERROR sync.worker.v2 failed for REQUEST_ID 9F42 after retry 17.\n\n"); + for i in 40..80 { + let _ = writeln!( + s, + "ordinary deployment progress line {i} with routine status information.\n" + ); + } + s +} + +fn html_document() -> String { + let mut body = String::from( + "Fixture

Fixture page

", + ); + for i in 0..120 { + let _ = write!( + body, + "

paragraph {i} with useful extracted text and repeated markup wrappers.

" + ); + } + body.push_str(""); + body +} + +fn assert_recoverable(text: &str, expected_original: &str) { + let tokens = cache::parse_markers(text); + assert_eq!(tokens.len(), 1, "expected one recovery marker in:\n{text}"); + let original = cache::retrieve(&tokens[0]).expect("footer token must be retrievable"); + assert_eq!(original, expected_original); +} + #[tokio::test] async fn off_profile_is_exact_passthrough() { common::install_test_config(); @@ -95,8 +162,8 @@ async fn full_profile_compacts_and_original_is_recoverable() { common::install_test_config(); let payload = big_json_rows(); let (text, stats) = compact_tool_output_with_policy( - "read_file", - Some(&json!({ "path": "services.json" })), + "http_request", + Some(&json!({ "url": "https://example.test/services" })), &payload, Some(0), AgentTokenjuiceCompression::Full, @@ -133,6 +200,105 @@ async fn full_profile_compacts_and_original_is_recoverable() { } } +#[tokio::test] +async fn diff_noise_compacts_recoverably_through_tool_hook() { + common::install_test_config(); + let payload = lockfile_diff(); + let (text, stats) = compact_tool_output_with_policy( + "shell", + Some(&json!({ "command": "git diff Cargo.lock" })), + &payload, + Some(0), + AgentTokenjuiceCompression::Full, + ) + .await; + + assert!(stats.applied, "expected DiffNoise, got {stats:?}"); + assert_eq!(stats.rule_id, "diff"); + assert!(text.contains("reason=lockfile"), "{text}"); + assert!(stats.compacted_bytes < stats.original_bytes); + assert_recoverable(&text, &payload); +} + +#[tokio::test] +async fn search_results_compact_recoverably_through_tool_hook() { + common::install_test_config(); + let payload = search_results(); + let (text, stats) = compact_tool_output_with_policy( + "grep", + Some(&json!({ "pattern": "needle_value" })), + &payload, + Some(0), + AgentTokenjuiceCompression::Full, + ) + .await; + + assert!(stats.applied, "expected search compressor, got {stats:?}"); + assert_eq!(stats.rule_id, "search"); + assert!(text.contains("needle_value"), "{text}"); + assert!(text.contains("search omitted"), "{text}"); + assert!(stats.compacted_bytes < stats.original_bytes); + assert_recoverable(&text, &payload); +} + +#[tokio::test] +async fn textcrusher_compacts_recoverably_through_tool_hook() { + common::install_test_config(); + let payload = status_report_text(); + let (text, stats) = compact_tool_output_with_policy( + "browser_extract_text", + Some(&json!({ "query": "sync.worker.v2 REQUEST_ID" })), + &payload, + Some(0), + AgentTokenjuiceCompression::Full, + ) + .await; + + assert!(stats.applied, "expected TextCrusher, got {stats:?}"); + assert_eq!(stats.rule_id, "textcrusher"); + assert!(text.contains("ERROR sync.worker.v2 failed"), "{text}"); + assert!(stats.compacted_bytes < stats.original_bytes); + assert_recoverable(&text, &payload); +} + +#[tokio::test] +async fn html_extract_compacts_recoverably_through_tool_hook() { + common::install_test_config(); + let payload = html_document(); + let (text, stats) = compact_tool_output_with_policy( + "http_request", + Some(&json!({ "url": "https://example.test/page", "path": "page.html" })), + &payload, + Some(0), + AgentTokenjuiceCompression::Full, + ) + .await; + + assert!(stats.applied, "expected HTML extraction, got {stats:?}"); + assert_eq!(stats.rule_id, "html"); + assert!(text.contains("Fixture page"), "{text}"); + assert!(!text.contains("