diff --git a/Native/pcai_core/Cargo.lock b/Native/pcai_core/Cargo.lock index 41113b4..b22fa54 100644 --- a/Native/pcai_core/Cargo.lock +++ b/Native/pcai_core/Cargo.lock @@ -1126,6 +1126,12 @@ dependencies = [ "libloading 0.9.0", ] +[[package]] +name = "daachorse" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f55d7153ba3b507595872a3874803f07a8a81d1e888abed8e5db7da0597d6e2" + [[package]] name = "darling" version = "0.11.0" @@ -4251,7 +4257,7 @@ dependencies = [ "serde_json", "tempfile", "thiserror 2.0.18", - "tokenizers 0.22.2", + "tokenizers 0.23.1", "tokio", "tower-http", "tracing", @@ -6233,7 +6239,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" dependencies = [ "fastrand", - "getrandom 0.3.4", + "getrandom 0.4.2", "once_cell", "rustix", "windows-sys 0.61.2", @@ -6467,6 +6473,39 @@ dependencies = [ "esaxx-rs", "fancy-regex 0.14.0", "getrandom 0.3.4", + "itertools 0.14.0", + "log", + "macro_rules_attribute 0.2.2", + "monostate", + "onig", + "paste", + "rand 0.9.3", + "rayon", + "rayon-cond 0.4.0", + "regex", + "regex-syntax 0.8.8", + "serde", + "serde_json", + "spm_precompiled", + "thiserror 2.0.18", + "unicode-normalization-alignments", + "unicode-segmentation", + "unicode_categories", +] + +[[package]] +name = "tokenizers" +version = "0.23.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44e5bea67576e04b6ff8564c5d9e09c2ef0cf476502245f2f120e497769d3112" +dependencies = [ + "ahash", + "compact_str", + "daachorse", + "dary_heap", + "derive_builder 0.20.2", + "esaxx-rs", + "getrandom 0.3.4", "indicatif 0.18.3", "itertools 0.14.0", "log", diff --git a/Native/pcai_core/pcai_media/Cargo.toml b/Native/pcai_core/pcai_media/Cargo.toml index 05f792d..c6a2912 100644 --- a/Native/pcai_core/pcai_media/Cargo.toml +++ b/Native/pcai_core/pcai_media/Cargo.toml @@ -24,7 +24,7 @@ candle-core = { version = "0.9" } candle-nn = "0.10" candle-transformers = "0.9" safetensors = "0.7" -tokenizers = "0.22" +tokenizers = "0.23" image = "0.25" rand = "0.9" # Disable ureq (sync HTTP client) to eliminate the ring crate dependency.