diff --git a/Cargo.lock b/Cargo.lock index 3aea7087d8..c4d05c915a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4393,6 +4393,7 @@ dependencies = [ "tempfile", "thiserror 2.0.18", "tinyagents", + "tinycortex", "tinyflows", "tinyplace", "tokio", @@ -4400,7 +4401,7 @@ dependencies = [ "tokio-stream", "tokio-tungstenite 0.24.0", "tokio-util", - "toml", + "toml 1.1.2+spec-1.1.0", "tower", "tracing", "tracing-appender", @@ -5015,7 +5016,7 @@ version = "3.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e67ba7e9b2b56446f1d419b1d807906278ffa1a658a8a5d8a39dcb1f5a78614f" dependencies = [ - "toml_edit", + "toml_edit 0.25.11+spec-1.1.0", ] [[package]] @@ -6126,6 +6127,15 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "serde_spanned" +version = "0.6.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf41e0cfaf7226dca15e8197172c295a782857fcb97fad1808a166870dee75a3" +dependencies = [ + "serde", +] + [[package]] name = "serde_spanned" version = "1.1.1" @@ -6712,6 +6722,27 @@ dependencies = [ "tokio", ] +[[package]] +name = "tinycortex" +version = "0.1.1" +dependencies = [ + "anyhow", + "async-trait", + "chrono", + "git2", + "parking_lot", + "rand 0.8.6", + "regex", + "rusqlite", + "serde", + "serde_json", + "sha2 0.10.9", + "thiserror 2.0.18", + "toml 0.8.23", + "uuid 1.23.1", + "walkdir", +] + [[package]] name = "tinyflows" version = "0.3.0" @@ -6942,6 +6973,18 @@ dependencies = [ "tokio-util", ] +[[package]] +name = "toml" +version = "0.8.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc1beb996b9d83529a9e75c17a1686767d148d70663143c7854d8b4a09ced362" +dependencies = [ + "serde", + "serde_spanned 0.6.9", + "toml_datetime 0.6.11", + "toml_edit 0.22.27", +] + [[package]] name = "toml" version = "1.1.2+spec-1.1.0" @@ -6950,11 +6993,20 @@ checksum = "81f3d15e84cbcd896376e6730314d59fb5a87f31e4b038454184435cd57defee" dependencies = [ "indexmap", "serde_core", - "serde_spanned", - "toml_datetime", + "serde_spanned 1.1.1", + "toml_datetime 1.1.1+spec-1.1.0", "toml_parser", "toml_writer", - "winnow", + "winnow 1.0.2", +] + +[[package]] +name = "toml_datetime" +version = "0.6.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22cddaf88f4fbc13c51aebbf5f8eceb5c7c5a9da2ac40a13519eb5b0a0e8f11c" +dependencies = [ + "serde", ] [[package]] @@ -6966,6 +7018,20 @@ dependencies = [ "serde_core", ] +[[package]] +name = "toml_edit" +version = "0.22.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41fe8c660ae4257887cf66394862d21dbca4a6ddd26f04a3560410406a2f819a" +dependencies = [ + "indexmap", + "serde", + "serde_spanned 0.6.9", + "toml_datetime 0.6.11", + "toml_write", + "winnow 0.7.15", +] + [[package]] name = "toml_edit" version = "0.25.11+spec-1.1.0" @@ -6973,9 +7039,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b59c4d22ed448339746c59b905d24568fcbb3ab65a500494f7b8c3e97739f2b" dependencies = [ "indexmap", - "toml_datetime", + "toml_datetime 1.1.1+spec-1.1.0", "toml_parser", - "winnow", + "winnow 1.0.2", ] [[package]] @@ -6984,9 +7050,15 @@ version = "1.1.2+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a2abe9b86193656635d2411dc43050282ca48aa31c2451210f4202550afb7526" dependencies = [ - "winnow", + "winnow 1.0.2", ] +[[package]] +name = "toml_write" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d99f8c9a7727884afe522e9bd5edbfc91a3312b36a77b5fb8926e4c31a41801" + [[package]] name = "toml_writer" version = "1.1.1+spec-1.1.0" @@ -8579,6 +8651,15 @@ version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650" +[[package]] +name = "winnow" +version = "0.7.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df79d97927682d2fd8adb29682d1140b343be4ac0f08fd68b7765d9c059d3945" +dependencies = [ + "memchr", +] + [[package]] name = "winnow" version = "1.0.2" @@ -9074,7 +9155,3 @@ checksum = "27bc9d5b815bc103f142aa054f561d9187d191692ec7c2d1e2b4737f8dbd7296" dependencies = [ "zune-core", ] - -[[patch.unused]] -name = "tinycortex" -version = "0.1.1" diff --git a/Cargo.toml b/Cargo.toml index d89aa3baf0..9203eb4ac6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -58,6 +58,15 @@ tinyflows = "0.3" # Durable graph checkpoints still use `SqlRunLedgerCheckpointer` until the # migration re-points those rows to the crate checkpointer. tinyagents = { version = "1.5.0", features = ["sqlite"] } +# TinyCortex — Rust core for the memory engine (store/chunks/tree/retrieval/ +# queue/ingest/score + long tail), vendored as a git submodule and patched +# below to `vendor/tinycortex`. OpenHuman's memory subsystem migrates onto this +# crate through the adapter seam in `src/openhuman/tinycortex/` (mirroring the +# tinyagents seam): engine logic in the crate; RPC, agent tools, live sync, +# security gating, and the global singleton stay host-side. rusqlite/git2 are +# aligned to the host pins (=0.40 / 0.21) so one bundled SQLite + one libgit2 +# link. Keep the version pin in lockstep with the submodule tag. +tinycortex = "0.1" # TokenJuice code compressor — AST-aware signature extraction. Optional (C build) # behind the default `tokenjuice-treesitter` feature; disabling it falls back to # the language-agnostic brace-depth heuristic. See src/openhuman/tokenjuice/compressors/code.rs. diff --git a/src/openhuman/mod.rs b/src/openhuman/mod.rs index 983f2d9fe8..975e43f58d 100644 --- a/src/openhuman/mod.rs +++ b/src/openhuman/mod.rs @@ -124,6 +124,7 @@ pub mod text_input; pub mod thread_goals; pub mod threads; pub mod tinyagents; +pub mod tinycortex; pub mod tinyflows; pub mod tinyplace; pub mod tls; diff --git a/src/openhuman/tinycortex/chat.rs b/src/openhuman/tinycortex/chat.rs new file mode 100644 index 0000000000..1e603395df --- /dev/null +++ b/src/openhuman/tinycortex/chat.rs @@ -0,0 +1,114 @@ +//! LLM chat seam — bridge OpenHuman's memory chat runtime onto the crate's +//! [`ChatProvider`] (W1). +//! +//! TinyCortex extracts entities/topics and summarises via an injected +//! `ChatProvider` (it never makes a network call). OpenHuman already owns a +//! memory LLM surface — `memory::chat::{ChatProvider, ChatPrompt}` with +//! `build_chat_provider(&Config)` routing through `openhuman::inference` +//! (provider selection, credit metering, usage accounting). This adapter wraps +//! that host provider and re-exposes it as the crate's `ChatProvider`, so the +//! engine's LLM entity extractor / summariser drive OpenHuman inference without +//! duplicating any routing. +//! +//! The two contracts are near-identical (`name` + async `chat_for_json`). The +//! only conversion is the prompt: the host `ChatPrompt.temperature` is `f64`, +//! the crate's is `f32`; every other field maps 1:1. + +use std::sync::Arc; + +use async_trait::async_trait; +use tinycortex::memory::score::extract::{ + ChatPrompt as CortexChatPrompt, ChatProvider as CortexChatProvider, +}; + +use crate::openhuman::config::Config; +use crate::openhuman::memory::chat::{ + build_chat_provider as build_host_chat_provider, ChatPrompt as HostChatPrompt, + ChatProvider as HostChatProvider, +}; + +/// Wraps an OpenHuman [`HostChatProvider`] as the crate's [`CortexChatProvider`]. +pub struct SeamChatProvider { + inner: Arc, +} + +impl SeamChatProvider { + /// Build the adapter over a host chat provider (already routed through + /// `openhuman::inference`). + pub fn new(inner: Arc) -> Self { + tracing::debug!( + provider = inner.name(), + "[memory] constructing tinycortex chat seam over memory::chat::ChatProvider" + ); + Self { inner } + } +} + +#[async_trait] +impl CortexChatProvider for SeamChatProvider { + fn name(&self) -> &str { + self.inner.name() + } + + async fn chat_for_json(&self, prompt: &CortexChatPrompt) -> anyhow::Result { + let host = HostChatPrompt { + system: prompt.system.clone(), + user: prompt.user.clone(), + // Crate temperature is f32; host takes f64. + temperature: f64::from(prompt.temperature), + kind: prompt.kind, + max_tokens: prompt.max_tokens, + }; + self.inner.chat_for_json(&host).await + } +} + +/// Build a crate [`CortexChatProvider`] from the host [`Config`], routed through +/// `openhuman::inference` — the entry point the seam's LLM entity extractor and +/// summariser construct from. +pub fn build_chat_provider(config: &Config) -> anyhow::Result> { + let host = build_host_chat_provider(config)?; + Ok(Arc::new(SeamChatProvider::new(host))) +} + +#[cfg(test)] +mod tests { + use super::*; + + /// Echoes the fields it received back as a JSON body so the test can assert + /// the crate->host prompt conversion (incl. the f32->f64 temperature). + struct EchoHostProvider; + + #[async_trait] + impl HostChatProvider for EchoHostProvider { + fn name(&self) -> &str { + "echo" + } + async fn chat_for_json(&self, prompt: &HostChatPrompt) -> anyhow::Result { + Ok(format!( + "system={};user={};temp={};kind={};max={:?}", + prompt.system, prompt.user, prompt.temperature, prompt.kind, prompt.max_tokens + )) + } + } + + #[tokio::test] + async fn converts_prompt_and_delegates_to_host_provider() { + let seam = SeamChatProvider::new(Arc::new(EchoHostProvider)); + assert_eq!(CortexChatProvider::name(&seam), "echo"); + + let prompt = CortexChatPrompt { + system: "sys".to_string(), + user: "usr".to_string(), + temperature: 0.5, + kind: "extract", + max_tokens: Some(64), + }; + let out = seam.chat_for_json(&prompt).await.unwrap(); + // Every field maps 1:1; temperature widens f32 0.5 -> f64 0.5. + assert_eq!( + out, + "system=sys;user=usr;temp=0.5;kind=extract;max=Some(64)" + ); + } +} diff --git a/src/openhuman/tinycortex/config.rs b/src/openhuman/tinycortex/config.rs new file mode 100644 index 0000000000..d45f3c3eb7 --- /dev/null +++ b/src/openhuman/tinycortex/config.rs @@ -0,0 +1,78 @@ +//! `Config` → [`tinycortex::memory::MemoryConfig`] mapping (W1). +//! +//! The crate's [`MemoryConfig`] is the single input every engine primitive +//! takes (`workspace`, embedding dims/model/strict, tree budgets, retrieval +//! weight profile, sync budget). This adapter derives it from OpenHuman's +//! host [`Config`] plus the resolved memory workspace root, so the rest of the +//! seam constructs engine calls from real product configuration. +//! +//! Field provenance: +//! - `workspace` ← the memory workspace root (same root `MemoryClient` opens). +//! - `embedding.dim` ← `config.memory.embedding_dimensions`. +//! - `embedding.model` ← `config.memory.embedding_model`. +//! - `embedding.strict` ← `config.memory_tree.embedding_strict` (when false the +//! engine tolerates an inert embedder and falls back to scope+recency rerank). +//! - `tree` / `retrieval` / `sync_budget` ← crate defaults, which already match +//! the host engine's constants (`INPUT_TOKEN_BUDGET = 50_000`, +//! `OUTPUT_TOKEN_BUDGET = 5_000`, `SUMMARY_FANOUT = 10`, +//! `DEFAULT_FLUSH_AGE_SECS = 604_800`). The `tree_policy.rs` flavour overlays +//! and per-source `WeightProfile` selection are layered on at call sites in +//! later workstreams; this base mapping is the W1 foundation. + +use std::path::PathBuf; + +use tinycortex::memory::config::EmbeddingConfig; +use tinycortex::memory::MemoryConfig; + +use crate::openhuman::config::Config; + +/// Build a [`MemoryConfig`] from the host [`Config`] and the resolved memory +/// workspace root. +/// +/// `workspace` is the directory under which the engine stores `chunks.db`, the +/// content vault, and the tree DBs — it must be the same root the host +/// `MemoryClient` opens so an existing user workspace is read in place (parity +/// is gated by the W3 golden-workspace harness). +pub fn memory_config_from(config: &Config, workspace: PathBuf) -> MemoryConfig { + let mut mc = MemoryConfig::new(workspace); + mc.embedding = EmbeddingConfig { + dim: config.memory.embedding_dimensions, + model: config.memory.embedding_model.clone(), + strict: config.memory_tree.embedding_strict, + }; + mc +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn maps_workspace_and_embedding_from_host_config() { + let mut config = Config::default(); + config.memory.embedding_dimensions = 1024; + config.memory.embedding_model = "embedding-v1".to_string(); + config.memory_tree.embedding_strict = true; + + let workspace = PathBuf::from("/tmp/openhuman/ws"); + let mc = memory_config_from(&config, workspace.clone()); + + assert_eq!(mc.workspace, workspace); + assert_eq!(mc.embedding.dim, 1024); + assert_eq!(mc.embedding.model, "embedding-v1"); + assert!(mc.embedding.strict); + } + + #[test] + fn tree_defaults_match_engine_constants() { + // The base mapping leaves tree budgets at the crate defaults, which are + // the host engine's own constants — asserted here so a crate-side change + // to those defaults surfaces as a failing parity test rather than a + // silent behaviour drift. + let mc = memory_config_from(&Config::default(), PathBuf::from("/tmp/ws")); + assert_eq!(mc.tree.input_token_budget, 50_000); + assert_eq!(mc.tree.output_token_budget, 5_000); + assert_eq!(mc.tree.summary_fanout, 10); + assert_eq!(mc.tree.flush_age_secs, 604_800); + } +} diff --git a/src/openhuman/tinycortex/embeddings.rs b/src/openhuman/tinycortex/embeddings.rs new file mode 100644 index 0000000000..c34c7f8439 --- /dev/null +++ b/src/openhuman/tinycortex/embeddings.rs @@ -0,0 +1,139 @@ +//! Embedding seam — bridge OpenHuman's [`EmbeddingProvider`] onto the crate's +//! two embedding traits (W1). +//! +//! OpenHuman owns the concrete providers (voyage / openai / cohere / ollama / +//! cloud / noop) and the `embeddings/factory.rs` construction policy (rate-limit +//! + retry). TinyCortex "never makes a network call" — it takes compute through +//! [`EmbeddingBackend`] (the vector store) and [`Embedder`] (retrieval / seal +//! scoring). This adapter wraps one `Arc` and re-exposes +//! it as both, so the engine drives OpenHuman embeddings without cloning any +//! provider logic. +//! +//! The two host and crate contracts are shape-identical (`name` / `model_id` / +//! `dimensions` / `signature` / async `embed`), and both use `anyhow::Result`, +//! so this is a near-pure pass-through. Critically, `signature()` delegates to +//! the provider so the persisted embedding-space signature +//! (`provider=…;model=…;dims=…`) stays byte-identical whether the store keys off +//! the crate backend or the raw provider (#1574 fidelity). + +use std::sync::Arc; + +use async_trait::async_trait; +use tinycortex::memory::score::embed::Embedder; +use tinycortex::memory::store::vectors::EmbeddingBackend; + +use crate::openhuman::embeddings::EmbeddingProvider; + +/// Wraps an OpenHuman [`EmbeddingProvider`] as the crate's [`EmbeddingBackend`] +/// (vector store) and [`Embedder`] (retrieval / seal scoring). +pub struct SeamEmbedder { + provider: Arc, +} + +impl SeamEmbedder { + /// Build the adapter over an OpenHuman embedding provider, preserving its + /// factory-configured rate-limit + retry policy. + pub fn new(provider: Arc) -> Self { + tracing::debug!( + provider = provider.name(), + model_id = provider.model_id(), + dimensions = provider.dimensions(), + signature = %provider.signature(), + "[memory] constructing tinycortex embedding seam over EmbeddingProvider" + ); + Self { provider } + } +} + +#[async_trait] +impl EmbeddingBackend for SeamEmbedder { + fn name(&self) -> &str { + self.provider.name() + } + + fn model_id(&self) -> &str { + self.provider.model_id() + } + + fn dimensions(&self) -> usize { + self.provider.dimensions() + } + + /// Delegate to the provider so the persisted signature is byte-identical to + /// the config-derived `active_embedding_signature` — a mismatch would split + /// one embedding space into two (#1574). + fn signature(&self) -> String { + self.provider.signature() + } + + async fn embed(&self, texts: &[&str]) -> anyhow::Result>> { + self.provider.embed(texts).await + } +} + +#[async_trait] +impl Embedder for SeamEmbedder { + fn name(&self) -> &'static str { + // The crate's `Embedder` requires a `'static` name (debug/diagnostics + // only); the provider's own `name()` is borrowed, so report a stable + // seam label rather than leaking a lifetime. + "openhuman-seam" + } + + async fn embed(&self, text: &str) -> anyhow::Result> { + self.provider.embed_one(text).await + } +} + +#[cfg(test)] +mod tests { + use super::*; + + struct FakeProvider; + + #[async_trait] + impl EmbeddingProvider for FakeProvider { + fn name(&self) -> &str { + "fake" + } + fn model_id(&self) -> &str { + "fake-model" + } + fn dimensions(&self) -> usize { + 3 + } + async fn embed(&self, texts: &[&str]) -> anyhow::Result>> { + Ok(texts + .iter() + .map(|t| vec![t.len() as f32, 0.0, 0.0]) + .collect()) + } + } + + #[tokio::test] + async fn backend_passes_through_metadata_and_signature() { + let seam = SeamEmbedder::new(Arc::new(FakeProvider)); + assert_eq!(EmbeddingBackend::name(&seam), "fake"); + assert_eq!(seam.model_id(), "fake-model"); + assert_eq!(seam.dimensions(), 3); + // Byte-identical to format_embedding_signature(name, model_id, dims). + assert_eq!( + EmbeddingBackend::signature(&seam), + "provider=fake;model=fake-model;dims=3" + ); + } + + #[tokio::test] + async fn backend_and_embedder_both_delegate_to_provider() { + let seam = SeamEmbedder::new(Arc::new(FakeProvider)); + + let batch = EmbeddingBackend::embed(&seam, &["ab", "cde"]) + .await + .unwrap(); + assert_eq!(batch, vec![vec![2.0, 0.0, 0.0], vec![3.0, 0.0, 0.0]]); + + let one = Embedder::embed(&seam, "abcd").await.unwrap(); + assert_eq!(one, vec![4.0, 0.0, 0.0]); + assert_eq!(Embedder::name(&seam), "openhuman-seam"); + } +} diff --git a/src/openhuman/tinycortex/mod.rs b/src/openhuman/tinycortex/mod.rs new file mode 100644 index 0000000000..09a7630f7f --- /dev/null +++ b/src/openhuman/tinycortex/mod.rs @@ -0,0 +1,52 @@ +//! `tinycortex` integration — run OpenHuman's memory engine on the published +//! [`tinycortex`](https://crates.io/crates/tinycortex) crate. +//! +//! OpenHuman's memory subsystem migrates onto the `tinycortex` crate (store / +//! chunks / tree / retrieval / queue / ingest / score + the long tail). This +//! module is the **adapter seam**, mirroring `src/openhuman/tinyagents/`: it +//! implements the crate's engine traits over OpenHuman services and derives the +//! engine's [`MemoryConfig`] from the host [`Config`]. Nothing here contains +//! engine logic — that lives in the crate. +//! +//! ## Ownership boundary (the seam contract) +//! +//! **Engine (crate):** content store + YAML vault, SQLite vectors/kv/entity +//! index, chunk lifecycle, summary trees, hybrid retrieval, scoring, the async +//! job model, ingest canonicalize/extract, and the diff/entities/graph/goals/ +//! archivist/tool-memory/conversations long tail. +//! +//! **Product (host, stays in OpenHuman):** JSON-RPC schemas/ops/read_rpc, agent +//! tools + `SecurityPolicy` gating, live sync (`memory_sync`), the event bus, +//! preferences, `source_scope` per-turn allowlist, redaction, the global +//! singleton + background queue worker, embedding/LLM **compute**, and the +//! host-retained `UnifiedMemory` namespace-document tier (episodic/event/ +//! segment/doc/graph/profile tables) plus the `wiki_git`/`obsidian` content +//! surfaces the crate deliberately excludes. +//! +//! The crate never makes a network call or owns a worker pool: LLM/embedding +//! compute is injected through `EmbeddingBackend`, `ChatProvider`, `Summariser`, +//! and `EntityExtractor`; the job queue is driven by the host worker loop via +//! `queue::run_once` / `drain_until_idle`. Those adapters live beside this file +//! (`embeddings.rs`, `chat.rs`, `queue_driver.rs`, `sinks.rs`, `bus.rs`) and are +//! added workstream by workstream. +//! +//! See `docs/tinycortex-migration-spec.md` for the full ownership split, +//! drift/gap/parity ledgers, and the workstream order. + +mod chat; +mod config; +mod embeddings; + +pub use chat::{build_chat_provider, SeamChatProvider}; +pub use config::memory_config_from; +pub use embeddings::SeamEmbedder; + +// Facade re-exports — the rest of the host imports memory-engine types through +// this one seam so consumer import paths stay stable as the internals flip to +// the crate (the type-unification decision, spec §0.5). `MemoryTaint` is the +// security-critical provenance type; it is proven byte-identical to the host's +// (fail-closed to `ExternalSync`) before re-exporting. +pub use tinycortex::memory::{ + MemoryCategory, MemoryConfig, MemoryEntry, MemoryTaint, NamespaceSummary, RecallOpts, + WeightProfile, +}; diff --git a/vendor/tinycortex b/vendor/tinycortex index d1a8c7be2b..33dda94305 160000 --- a/vendor/tinycortex +++ b/vendor/tinycortex @@ -1 +1 @@ -Subproject commit d1a8c7be2babc8fff7a72ed93861f459f3d6fa58 +Subproject commit 33dda943053e61ef585fc39647cf1854344b6323