diff --git a/CMakeLists.txt b/CMakeLists.txt index 62ca12cf..f1eaa8fc 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -19,6 +19,11 @@ project(Tokenizers) option(TOKENIZERS_BUILD_TEST "Build tests" OFF) option(TOKENIZERS_BUILD_TOOLS "Build tools" OFF) option(TOKENIZERS_BUILD_PYTHON "Build Python bindings" OFF) +option( + TOKENIZERS_BUILD_HF_RUST_TOKENIZER + "Build the opt-in Hugging Face .tok tokenizer backend" OFF +) +option(TOKENIZERS_OPTIMIZE_SIZE "Optimize optional tokenizer backends for size" OFF) option(SUPPORT_REGEX_LOOKAHEAD "Support regex lookahead patterns (requires PCRE2)" OFF ) @@ -201,6 +206,10 @@ endif() # Installation rules include(GNUInstallDirs) +if(TOKENIZERS_BUILD_HF_RUST_TOKENIZER) + add_subdirectory(rust_tokenizer) +endif() + if(NOT TOKENIZERS_BUILD_PYTHON) # Install the library and its dependencies install( diff --git a/cmake/tokenizers-config.cmake.in b/cmake/tokenizers-config.cmake.in index 1e9f87b2..6a075dd5 100644 --- a/cmake/tokenizers-config.cmake.in +++ b/cmake/tokenizers-config.cmake.in @@ -25,6 +25,25 @@ endif() find_dependency(re2 REQUIRED) find_dependency(absl REQUIRED) +# The optional .tok backend is a C++ archive with a Rust static-library +# dependency. Define the latter before importing the exported targets when the +# archive is present. Merely finding the package does not link either target. +if(WIN32) + set(_TOKENIZERS_HF_FFI_NAME "tokenizers_hf_ffi.lib") +else() + set(_TOKENIZERS_HF_FFI_NAME "libtokenizers_hf_ffi.a") +endif() +set(_TOKENIZERS_HF_FFI "${TOKENIZERS_LIBDIR}/${_TOKENIZERS_HF_FFI_NAME}") +if(EXISTS "${_TOKENIZERS_HF_FFI}" AND NOT TARGET tokenizers_hf_ffi) + if(UNIX AND NOT APPLE) + find_dependency(Threads) + endif() + add_library(tokenizers_hf_ffi STATIC IMPORTED) + set_target_properties( + tokenizers_hf_ffi PROPERTIES IMPORTED_LOCATION "${_TOKENIZERS_HF_FFI}" + ) +endif() + # Include the exported targets file include("${CMAKE_CURRENT_LIST_DIR}/tokenizers-targets.cmake") diff --git a/include/pytorch/tokenizers/rust_hf_tokenizer.h b/include/pytorch/tokenizers/rust_hf_tokenizer.h new file mode 100644 index 00000000..b75e8fc7 --- /dev/null +++ b/include/pytorch/tokenizers/rust_hf_tokenizer.h @@ -0,0 +1,56 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +#pragma once + +#include +#include +#include +#include +#include +#include + +#include +#include + +namespace tokenizers { + +class RustHFTokenizer final : public Tokenizer { + public: + RustHFTokenizer(); + ~RustHFTokenizer() override; + + Error load(const std::string& tokenizer_path) override; + Result id_to_piece(uint64_t token) const override; + Result piece_to_id(const std::string& text) const override; + Result> encode( + const std::string& input, + int8_t bos = 0, + int8_t eos = 0) const override; + Result decode( + uint64_t prev_token, + uint64_t token, + bool skip_special_tokens = false) const override; + + private: + using TokenMap = detail::StringIntegerMap<>; + + struct RustHandleDeleter { + void operator()(void* handle) const; + }; + + Error load_metadata(const void* handle); + + std::unique_ptr handle_; + std::optional token_map_; + std::optional added_token_map_; + std::unordered_set special_token_ids_; + bool byte_level_ = false; +}; + +} // namespace tokenizers diff --git a/rust_tokenizer/.gitignore b/rust_tokenizer/.gitignore new file mode 100644 index 00000000..b83d2226 --- /dev/null +++ b/rust_tokenizer/.gitignore @@ -0,0 +1 @@ +/target/ diff --git a/rust_tokenizer/CMakeLists.txt b/rust_tokenizer/CMakeLists.txt new file mode 100644 index 00000000..ee219656 --- /dev/null +++ b/rust_tokenizer/CMakeLists.txt @@ -0,0 +1,79 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +find_program(CARGO_EXECUTABLE cargo REQUIRED) + +set(_rust_profile release) +if(TOKENIZERS_OPTIMIZE_SIZE) + set(_rust_profile minsize) +endif() + +set(_cargo_target_dir ${CMAKE_CURRENT_BINARY_DIR}/cargo-target) +string( + CONCAT + _rust_library_name + ${CMAKE_STATIC_LIBRARY_PREFIX} + tokenizers_hf_ffi + ${CMAKE_STATIC_LIBRARY_SUFFIX} +) +set(_rust_library ${_cargo_target_dir}/${_rust_profile}/${_rust_library_name}) +file(GLOB_RECURSE _rust_sources CONFIGURE_DEPENDS + ${CMAKE_CURRENT_SOURCE_DIR}/src/*.rs +) + +add_custom_command( + OUTPUT ${_rust_library} + COMMAND + ${CMAKE_COMMAND} -E env CARGO_TARGET_DIR=${_cargo_target_dir} + CARGO_ENCODED_RUSTFLAGS=-Crelocation-model=pic + ${CARGO_EXECUTABLE} build --locked --manifest-path + ${CMAKE_CURRENT_SOURCE_DIR}/Cargo.toml --profile ${_rust_profile} + DEPENDS ${_rust_sources} ${CMAKE_CURRENT_SOURCE_DIR}/Cargo.toml + ${CMAKE_CURRENT_SOURCE_DIR}/Cargo.lock + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} + VERBATIM +) + +add_custom_target(tokenizers_hf_ffi_build DEPENDS ${_rust_library}) +add_library(tokenizers_hf_ffi STATIC IMPORTED GLOBAL) +set_target_properties( + tokenizers_hf_ffi PROPERTIES IMPORTED_LOCATION ${_rust_library} +) +add_dependencies(tokenizers_hf_ffi tokenizers_hf_ffi_build) + +add_library(tokenizers_hf_rust_tokenizer STATIC ../src/rust_hf_tokenizer.cpp) +add_library(tokenizers::hf_rust_tokenizer ALIAS tokenizers_hf_rust_tokenizer) +set_target_properties( + tokenizers_hf_rust_tokenizer PROPERTIES EXPORT_NAME hf_rust_tokenizer +) +target_include_directories( + tokenizers_hf_rust_tokenizer + PUBLIC + $ + $ +) +target_compile_features(tokenizers_hf_rust_tokenizer PUBLIC cxx_std_17) +target_link_libraries( + tokenizers_hf_rust_tokenizer + PRIVATE tokenizers_hf_ffi +) + +if(APPLE) + target_link_libraries(tokenizers_hf_rust_tokenizer PRIVATE iconv) +elseif(UNIX) + find_package(Threads REQUIRED) + target_link_libraries( + tokenizers_hf_rust_tokenizer + PRIVATE Threads::Threads ${CMAKE_DL_LIBS} m rt util + ) +endif() + +install( + TARGETS tokenizers_hf_rust_tokenizer + EXPORT tokenizers-targets + ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} +) +install(FILES ${_rust_library} DESTINATION ${CMAKE_INSTALL_LIBDIR}) diff --git a/rust_tokenizer/Cargo.lock b/rust_tokenizer/Cargo.lock new file mode 100644 index 00000000..87eca543 --- /dev/null +++ b/rust_tokenizer/Cargo.lock @@ -0,0 +1,1456 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "ahash" +version = "0.8.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75" +dependencies = [ + "cfg-if", + "getrandom 0.3.4", + "once_cell", + "version_check", + "zerocopy", +] + +[[package]] +name = "aho-corasick" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c982642fa9e8606056828ee9a8505737230110bb1099153c79efe865c59d12ba" +dependencies = [ + "memchr", +] + +[[package]] +name = "allocator-api2" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" + +[[package]] +name = "anes" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299" + +[[package]] +name = "anstyle" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000" + +[[package]] +name = "anyhow" +version = "1.0.104" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" + +[[package]] +name = "arbitrary-chunks" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ad8689a486416c401ea15715a4694de30054248ec627edbf31f49cb64ee4086" + +[[package]] +name = "atomsplit" +version = "0.1.0" +source = "git+https://github.com/huggingface/tokenizers?rev=054bdf469b2cf416c0da953923d88c82f4d765b5#054bdf469b2cf416c0da953923d88c82f4d765b5" +dependencies = [ + "memchr", +] + +[[package]] +name = "autocfg" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" + +[[package]] +name = "bitflags" +version = "2.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b588b76d00fde79687d7646a9b5bdf3cc0f655e0bbd080335a95d7e96f3587da" + +[[package]] +name = "bitvec" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddcec3d12c579d40898fe0a9a358a803c23e9c52ca3c425707f81c9436211837" +dependencies = [ + "funty", + "radium", + "tap", + "wyz", +] + +[[package]] +name = "block-pseudorand" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2097358495d244a0643746f4d13eedba4608137008cf9dec54e53a3b700115a6" +dependencies = [ + "chiapos-chacha8", + "nanorand", +] + +[[package]] +name = "bumpalo" +version = "3.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" + +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + +[[package]] +name = "cacheline-ef" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af737c6c59cb018ecbe6472cbdf86d39c59d78252febfe311953a991b6e4ed85" +dependencies = [ + "common_traits", + "mem_dbg 0.3.4", +] + +[[package]] +name = "cast" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" + +[[package]] +name = "castaway" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dec551ab6e7578819132c713a93c022a05d60159dc86e7a7050223577484c55a" +dependencies = [ + "rustversion", +] + +[[package]] +name = "cc" +version = "1.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ad534f4357a5264cce5019c989cf66a4f0dc4e0d1b1d15f8aacec0ff7360273" +dependencies = [ + "find-msvc-tools", + "shlex", +] + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "chacha20" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65c35e4b699c7e15ccbe7ee35c005e4fc0a278d22238a2857e6ce2dadeda1b06" +dependencies = [ + "cfg-if", + "cpufeatures", + "rand_core 0.10.1", +] + +[[package]] +name = "chiapos-chacha8" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33f8be573a85f6c2bc1b8e43834c07e32f95e489b914bf856c0549c3c269cd0a" +dependencies = [ + "rayon", +] + +[[package]] +name = "ciborium" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42e69ffd6f0917f5c029256a24d0161db17cea3997d185db0d35926308770f0e" +dependencies = [ + "ciborium-io", + "ciborium-ll", + "serde", +] + +[[package]] +name = "ciborium-io" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05afea1e0a06c9be33d539b876f1ce3692f4afea2cb41f740e7743225ed1c757" + +[[package]] +name = "ciborium-ll" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57663b653d948a338bfb3eeba9bb2fd5fcfaecb9e199e87e1eda4d9e8b240fd9" +dependencies = [ + "ciborium-io", + "half", +] + +[[package]] +name = "clap" +version = "4.6.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "473c7e07f409a8d772161724aa8db6a765a2532a70f9667eeb7b49d3d02fbdca" +dependencies = [ + "clap_builder", +] + +[[package]] +name = "clap_builder" +version = "4.6.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b48fea5a88e9ae728a2dcbedbfc0e730f7d60da42e1cb049a83c9fb8b789889" +dependencies = [ + "anstyle", + "clap_lex", +] + +[[package]] +name = "clap_lex" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" + +[[package]] +name = "colored" +version = "3.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "faf9468729b8cbcea668e36183cb69d317348c2e08e994829fb56ebfdfbaac34" +dependencies = [ + "windows-sys", +] + +[[package]] +name = "common_traits" +version = "0.11.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fda9ae1f26adcae83adb2e92f69cf59421f2a277a942f49f8e59f2fcbd7cf062" +dependencies = [ + "anyhow", + "half", + "impl-tools", +] + +[[package]] +name = "compact_str" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9dfdd1c2274d9aa354115b09dc9a901d6c5576818cdf70d14cae2bdb47df00ab" +dependencies = [ + "castaway", + "cfg-if", + "itoa", + "rustversion", + "ryu", + "static_assertions", +] + +[[package]] +name = "cpufeatures" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ca28b0ae3115b884660db4118d803791fd6756b6e88f39c0f3f7859060d7566" +dependencies = [ + "libc", +] + +[[package]] +name = "criterion" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2b12d017a929603d80db1831cd3a24082f8137ce19c69e6447f54f5fc8d692f" +dependencies = [ + "anes", + "cast", + "ciborium", + "clap", + "criterion-plot", + "is-terminal", + "itertools 0.10.5", + "num-traits", + "once_cell", + "oorandom", + "plotters", + "rayon", + "regex", + "serde", + "serde_derive", + "serde_json", + "tinytemplate", + "walkdir", +] + +[[package]] +name = "criterion-plot" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b50826342786a51a89e2da3a28f1c32b06e387201bc2d19791f622c673706b1" +dependencies = [ + "cast", + "itertools 0.10.5", +] + +[[package]] +name = "crossbeam-deque" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5181e0de7b61eb03a81e347d6dd8797bae9da5146707b51077e2d71a54ec0ceb" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d6914041f254d6e9176c01941b21115dcfb7089e55135a35411081bd106ef3f" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61803da095bee82a81bb1a452ecc25d3b2f1416d1897eb86430c6159ef717c17" + +[[package]] +name = "crunchy" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" + +[[package]] +name = "daachorse" +version = "3.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5614204febbc33cc07a2806aa6440b904ac012b68eecc37f4493ea4a76455a3d" + +[[package]] +name = "darling" +version = "0.20.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc7f46116c46ff9ab3eb1597a45688b6715c6e628b5c133e288e709a29bcb4ee" +dependencies = [ + "darling_core", + "darling_macro", +] + +[[package]] +name = "darling_core" +version = "0.20.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d00b9596d185e565c2207a0b01f8bd1a135483d02d9b7b0a54b11da8d53412e" +dependencies = [ + "fnv", + "ident_case", + "proc-macro2", + "quote", + "strsim", + "syn 2.0.119", +] + +[[package]] +name = "darling_macro" +version = "0.20.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc34b93ccb385b40dc71c6fceac4b2ad23662c7eeb248cf10d529b7e055b6ead" +dependencies = [ + "darling_core", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "dary_heap" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b1e3a325bc115f096c8b77bbf027a7c2592230e70be2d985be950d3d5e60ebe" + +[[package]] +name = "derive_builder" +version = "0.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "507dfb09ea8b7fa618fcf76e953f4f5e192547945816d5358edffe39f6f94947" +dependencies = [ + "derive_builder_macro", +] + +[[package]] +name = "derive_builder_core" +version = "0.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d5bcf7b024d6835cfb3d473887cd966994907effbe9227e8c8219824d06c4e8" +dependencies = [ + "darling", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "derive_builder_macro" +version = "0.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab63b0e2bf4d5928aff72e83a7dace85d7bba5fe12dcc3c5a572d78caffd3f3c" +dependencies = [ + "derive_builder_core", + "syn 2.0.119", +] + +[[package]] +name = "either" +version = "1.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "252afb9ae5eaa683babdc6a068b3f5726eb19e05070c731f9b2a23a7c3e8ed34" + +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys", +] + +[[package]] +name = "pytorch-tokenizers-hf-ffi" +version = "0.1.0" +dependencies = [ + "tk-encode", + "tk-serialization", +] + +[[package]] +name = "fastrand" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" + +[[package]] +name = "find-msvc-tools" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d45db016d36b838f563236e9193d0ee6ce38f3f68b6c94e914b4929c96bbb890" + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "foldhash" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" + +[[package]] +name = "funty" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6d5a32815ae3f33302d95fdcb2ce17862f8c65363dcfd29360480ba1001fc9c" + +[[package]] +name = "futures-core" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92d699e522242e69e3003b94ecc1f960f3a5e015aa7c5d7486e65ad01dd94f5e" + +[[package]] +name = "futures-task" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd417de3d1d015fc3bfd2b1ea46dfc7bab72ef86f1cc7cc9c78e728b34a6d1fd" + +[[package]] +name = "futures-util" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d50a92467f8ba5dd6e3ee5d4bd04d73ab2e4e1c44474a0674821dfce14b79bc" +dependencies = [ + "futures-core", + "futures-task", + "pin-project-lite", + "slab", +] + +[[package]] +name = "fxhash" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c31b6d751ae2c7f11320402d34e41349dd1016f8d5d45e48c4312bc8625af50c" +dependencies = [ + "byteorder", +] + +[[package]] +name = "getrandom" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" +dependencies = [ + "cfg-if", + "libc", + "r-efi 5.3.0", + "wasip2", +] + +[[package]] +name = "getrandom" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" +dependencies = [ + "cfg-if", + "libc", + "r-efi 6.0.0", + "rand_core 0.10.1", +] + +[[package]] +name = "half" +version = "2.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b" +dependencies = [ + "cfg-if", + "crunchy", + "zerocopy", +] + +[[package]] +name = "hashbrown" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" +dependencies = [ + "allocator-api2", + "equivalent", + "foldhash", +] + +[[package]] +name = "hermit-abi" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e17592d60ebacc7d5e169f4663c5f84f9161cc90328abcfe8456f41e4dfcb284" + +[[package]] +name = "ident_case" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" + +[[package]] +name = "impl-tools" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ae95c9095c2f1126d7db785955c73cdc5fc33e7c3fa911bd4a42931672029a7" +dependencies = [ + "autocfg", + "impl-tools-lib", + "proc-macro-error2", + "syn 2.0.119", +] + +[[package]] +name = "impl-tools-lib" +version = "0.11.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab699036df31c1f7d3561bfa6e9cb9bc3bb0fd2e2cd9bf121c31cb961d049ddf" +dependencies = [ + "proc-macro-error2", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "is-terminal" +version = "0.4.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46" +dependencies = [ + "hermit-abi", + "libc", + "windows-sys", +] + +[[package]] +name = "itertools" +version = "0.10.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473" +dependencies = [ + "either", +] + +[[package]] +name = "itertools" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285" +dependencies = [ + "either", +] + +[[package]] +name = "itertools" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b4baf93f58d4425749ca49a51c50ebab072c5df6994d08fed93541c331481dc" +dependencies = [ + "either", +] + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "js-sys" +version = "0.3.104" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e0c1080212aad755ea003d18543e8768dd432c48819efd73a7bf1e39b7a5a3a" +dependencies = [ + "cfg-if", + "futures-util", + "wasm-bindgen", +] + +[[package]] +name = "libc" +version = "0.2.189" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" + +[[package]] +name = "linux-raw-sys" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" + +[[package]] +name = "log" +version = "0.4.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6" + +[[package]] +name = "macro_rules_attribute" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b3ae8f6d608c795738406608304d30a2dfbdc8e58e44f7ba43236da5208ded3c" +dependencies = [ + "macro_rules_attribute-proc_macro", + "pastey", +] + +[[package]] +name = "macro_rules_attribute-proc_macro" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc04a4c58212d57930a24bf47d3fa87485264a3a054e9c10e042eb373573ad3c" + +[[package]] +name = "mem_dbg" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "728cc9dc97593cd22f7bc81fbef70a2d391d7a9a855e7d658b653318124a6cf0" +dependencies = [ + "bitflags", + "mem_dbg-derive 0.2.1", +] + +[[package]] +name = "mem_dbg" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b48a1086c746f4ee6ca5cb0acf856a14709bc4d2d20e03db150a12ddf2269e6d" +dependencies = [ + "bitflags", + "hashbrown", + "mem_dbg-derive 0.3.4", +] + +[[package]] +name = "mem_dbg-derive" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d84f40c93b0508d5565db79a814d02d5b2545967205ce44be211592aafa34d6c" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "mem_dbg-derive" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eb910efe8da52f13da727170e352e50a1764579a6fb1065d00d9556da19c79ac" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + +[[package]] +name = "monostate" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3341a273f6c9d5bef1908f17b7267bbab0e95c9bf69a0d4dcf8e9e1b2c76ef67" +dependencies = [ + "monostate-impl", + "serde", + "serde_core", +] + +[[package]] +name = "monostate-impl" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4db6d5580af57bf992f59068d4ea26fd518574ff48d7639b255a36f9de6e7e9" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "nanorand" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "729eb334247daa1803e0a094d0a5c55711b85571179f5ec6e53eccfdf7008958" + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", +] + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "oorandom" +version = "11.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6790f58c7ff633d8771f42965289203411a5e5c68388703c06e14f24770b41e" + +[[package]] +name = "partition" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "947f833aaa585cf12b8ec7c0476c98784c49f33b861376ffc84ed92adebf2aba" + +[[package]] +name = "paste" +version = "1.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" + +[[package]] +name = "pastey" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ee67f1008b1ba2321834326597b8e186293b049a023cdef258527550b9935b4" + +[[package]] +name = "pin-project-lite" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" + +[[package]] +name = "plotters" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5aeb6f403d7a4911efb1e33402027fc44f29b5bf6def3effcc22d7bb75f2b747" +dependencies = [ + "num-traits", + "plotters-backend", + "plotters-svg", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "plotters-backend" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df42e13c12958a16b3f7f4386b9ab1f3e7933914ecea48da7139435263a4172a" + +[[package]] +name = "plotters-svg" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51bae2ac328883f7acdfea3d66a7c35751187f870bc81f94563733a154d7a670" +dependencies = [ + "plotters-backend", +] + +[[package]] +name = "ppv-lite86" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +dependencies = [ + "zerocopy", +] + +[[package]] +name = "prefetch-index" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9057806a8d77d67bccdc0f542db43737a6f19ada3efab2adc63277feea27310f" + +[[package]] +name = "proc-macro-error-attr2" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96de42df36bb9bba5542fe9f1a054b8cc87e172759a1868aa05c1f3acc89dfc5" +dependencies = [ + "proc-macro2", + "quote", +] + +[[package]] +name = "proc-macro-error2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11ec05c52be0a07b08061f7dd003e7d7092e0472bc731b4af7bb1ef876109802" +dependencies = [ + "proc-macro-error-attr2", + "proc-macro2", + "quote", +] + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "ptr_hash" +version = "2.0.2" +source = "git+https://github.com/ArthurZucker/PtrHash?rev=fff63a67eec9b48b693b0d3d2b253db78c0b9858#fff63a67eec9b48b693b0d3d2b253db78c0b9858" +dependencies = [ + "bitvec", + "cacheline-ef", + "colored", + "fastrand", + "fxhash", + "itertools 0.15.0", + "log", + "mem_dbg 0.4.4", + "prefetch-index", + "rand 0.10.2", + "rand_chacha 0.10.0", + "rdst", + "sucds", + "tempfile", + "xxhash-rust", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" + +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + +[[package]] +name = "radium" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc33ff2d4973d518d823d61aa239014831e521c75da58e3df4840d3f47749d09" + +[[package]] +name = "rand" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9ef1d0d795eb7d84685bca4f72f3649f064e6641543d3a8c415898726a57b41" +dependencies = [ + "rand_chacha 0.9.0", + "rand_core 0.9.5", +] + +[[package]] +name = "rand" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7f5fa3a058cd35567ef9bfa5e75732bee0f9e4c55fa90477bef2dfcdbc4be80" +dependencies = [ + "chacha20", + "getrandom 0.4.3", + "rand_core 0.10.1", +] + +[[package]] +name = "rand_chacha" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" +dependencies = [ + "ppv-lite86", + "rand_core 0.9.5", +] + +[[package]] +name = "rand_chacha" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e6af7f3e25ded52c41df4e0b1af2d047e45896c2f3281792ed68a1c243daedb" +dependencies = [ + "ppv-lite86", + "rand_core 0.10.1", +] + +[[package]] +name = "rand_core" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76afc826de14238e6e8c374ddcc1fa19e374fd8dd986b0d2af0d02377261d83c" +dependencies = [ + "getrandom 0.3.4", +] + +[[package]] +name = "rand_core" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69" + +[[package]] +name = "rayon" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb39b166781f92d482534ef4b4b1b2568f42613b53e5b6c160e24cfbfa30926d" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91" +dependencies = [ + "crossbeam-deque", + "crossbeam-utils", +] + +[[package]] +name = "rdst" +version = "0.20.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e7970b4e577b76a96d5e56b5f6662b66d1a4e1f5bb026ee118fc31b373c2752" +dependencies = [ + "arbitrary-chunks", + "block-pseudorand", + "criterion", + "partition", + "tikv-jemallocator", + "voracious_radix_sort", +] + +[[package]] +name = "regex" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad8553b9b26413251cbf30e620595c7a41b3887f03da04579c0e6b0d6a06b4b2" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" + +[[package]] +name = "rustix" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys", + "windows-sys", +] + +[[package]] +name = "rustversion" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" + +[[package]] +name = "ryu" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" + +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "serde" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.4", +] + +[[package]] +name = "serde_json" +version = "1.0.151" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "shlex" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" + +[[package]] +name = "slab" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" + +[[package]] +name = "static_assertions" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" + +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + +[[package]] +name = "sucds" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd324eaa05be64f105ea5269bb8aabd70e5dd57fa5c673b167f451b07d6c0dcd" +dependencies = [ + "anyhow", + "num-traits", +] + +[[package]] +name = "syn" +version = "2.0.119" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "3.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6275cddf4610d1775e6d1fe9469b2e77d0f39fd98fb7450901b821e0c53649f" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "tap" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" + +[[package]] +name = "tempfile" +version = "3.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" +dependencies = [ + "fastrand", + "getrandom 0.4.3", + "once_cell", + "rustix", + "windows-sys", +] + +[[package]] +name = "thiserror" +version = "2.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec86235f5fcc2a73650310756d2ac5b138a5780bbbdfae3eeccec992c435ba4f" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc04cd3e1236dd4a98afca4569f2deb3f120e5422a4023be2cb683f8486292af" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.4", +] + +[[package]] +name = "tikv-jemalloc-sys" +version = "0.5.4+5.3.0-patched" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9402443cb8fd499b6f327e40565234ff34dbda27460c5b47db0db77443dd85d1" +dependencies = [ + "cc", + "libc", +] + +[[package]] +name = "tikv-jemallocator" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "965fe0c26be5c56c94e38ba547249074803efd52adfb66de62107d95aab3eaca" +dependencies = [ + "libc", + "tikv-jemalloc-sys", +] + +[[package]] +name = "tinytemplate" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be4d6b5f19ff7664e8c98d03e2139cb510db9b0a60b55f8e8709b689d939b6bc" +dependencies = [ + "serde", + "serde_json", +] + +[[package]] +name = "tk-encode" +version = "0.23.2-dev.0" +source = "git+https://github.com/huggingface/tokenizers?rev=054bdf469b2cf416c0da953923d88c82f4d765b5#054bdf469b2cf416c0da953923d88c82f4d765b5" +dependencies = [ + "ahash", + "atomsplit", + "compact_str", + "daachorse", + "dary_heap", + "derive_builder", + "getrandom 0.3.4", + "itertools 0.14.0", + "log", + "macro_rules_attribute", + "memchr", + "monostate", + "paste", + "ptr_hash", + "rand 0.9.5", + "regex", + "thiserror", + "tk-serialization", + "unicode-segmentation", + "unicode_categories", + "yada", +] + +[[package]] +name = "tk-serialization" +version = "0.23.2-dev.0" +source = "git+https://github.com/huggingface/tokenizers?rev=054bdf469b2cf416c0da953923d88c82f4d765b5#054bdf469b2cf416c0da953923d88c82f4d765b5" + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "unicode-segmentation" +version = "1.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6f5d3c3b1bf09027a88a6bc961fc00497d651009560b5463668dc81b0fa87a8" + +[[package]] +name = "unicode_categories" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39ec24b3121d976906ece63c9daad25b85969647682eee313cb5779fdd69e14e" + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "voracious_radix_sort" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "446e7ffcb6c27a71d05af7e51ef2ee5b71c48424b122a832f2439651e1914899" +dependencies = [ + "rayon", +] + +[[package]] +name = "walkdir" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" +dependencies = [ + "same-file", + "winapi-util", +] + +[[package]] +name = "wasip2" +version = "1.0.4+wasi-0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b67efb37e106e55ce722a510d6b5f9c17f083e5fc79afc2badeb12cc313d9487" +dependencies = [ + "wit-bindgen", +] + +[[package]] +name = "wasm-bindgen" +version = "0.2.127" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b70935747edd64d89de3efa29d73789b806c15798f8e7dca4d8ac356b50ce70" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.127" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77775f8f3f7217702089053b94958f8f54061a3f663417df76e19cbdcca29bc1" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.127" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e11d33f857dc2fb11b8bc75aee111aa9cbeb12cd9f25efd3d4c2a3dd4e235284" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn 2.0.119", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.127" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ef64dbcc55df09c7e5a46182d181c2cfa3e925f3da937ea764728b4bbb9dcbf" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "web-sys" +version = "0.3.104" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c435338968042f4f59a557f690a253676d47ce13ceb55d70100e7facf6620a30" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "winapi-util" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" +dependencies = [ + "windows-sys", +] + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "wit-bindgen" +version = "0.57.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" + +[[package]] +name = "wyz" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05f360fc0b24296329c78fda852a1e9ae82de9cf7b27dae4b7f62f118f77b9ed" +dependencies = [ + "tap", +] + +[[package]] +name = "xxhash-rust" +version = "0.8.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aee1b19627c7c60102ab80d3a9cbe18de90bfe03bfa6c3715447681f0e8c8af6" + +[[package]] +name = "yada" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53c3bb06259642a57b4ea1bf2a8260f7d94b7b78a096c46f193318918d925f61" + +[[package]] +name = "zerocopy" +version = "0.8.56" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "556764e583adb45a9f8d413c2a147fa7e8d821e48e12b14fd560b607998b75eb" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.56" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2ab42fc20575779bd240faa45f94a74256f755c0fa9e89f0ede20d91d0cdfc1" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "zmij" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" diff --git a/rust_tokenizer/Cargo.toml b/rust_tokenizer/Cargo.toml new file mode 100644 index 00000000..167d7de8 --- /dev/null +++ b/rust_tokenizer/Cargo.toml @@ -0,0 +1,33 @@ +[package] +name = "pytorch-tokenizers-hf-ffi" +version = "0.1.0" +edition = "2024" +publish = false + +[lib] +name = "tokenizers_hf_ffi" +crate-type = ["staticlib"] + +[dependencies] +tk-encode = { git = "https://github.com/huggingface/tokenizers", rev = "054bdf469b2cf416c0da953923d88c82f4d765b5", default-features = false } +tk-serialization = { git = "https://github.com/huggingface/tokenizers", rev = "054bdf469b2cf416c0da953923d88c82f4d765b5" } + +[dev-dependencies] +tk-serialization = { git = "https://github.com/huggingface/tokenizers", rev = "054bdf469b2cf416c0da953923d88c82f4d765b5", features = ["write"] } + +# The `.tok` branch relies on PtrHash's dependency gates until they are +# available in a crates.io release. Pin the same revision rather than following +# the mutable upstream branch. +[patch.crates-io] +ptr_hash = { git = "https://github.com/ArthurZucker/PtrHash", rev = "fff63a67eec9b48b693b0d3d2b253db78c0b9858" } + +[profile.release] +codegen-units = 1 +lto = "fat" +opt-level = 3 +strip = true + +[profile.minsize] +inherits = "release" +opt-level = "z" +panic = "abort" diff --git a/rust_tokenizer/README.md b/rust_tokenizer/README.md new file mode 100644 index 00000000..63f82a04 --- /dev/null +++ b/rust_tokenizer/README.md @@ -0,0 +1,45 @@ +# Experimental Hugging Face Rust tokenizer backend + +This backend is opt-in and disabled by default. Enable it with +`-DTOKENIZERS_BUILD_HF_RUST_TOKENIZER=ON`. When disabled, CMake does not enter +this directory, invoke Cargo, build the Rust archive, or compile the C++ +bridge, so the default tokenizer library is unchanged. ExecuTorch users can +enable the same target with both `-DEXECUTORCH_BUILD_EXTENSION_LLM=ON` and +`-DEXECUTORCH_BUILD_HF_RUST_TOKENIZER=ON`. +Use `-DTOKENIZERS_OPTIMIZE_SIZE=ON` for the Rust `minsize` profile; ExecuTorch +forwards `EXECUTORCH_OPTIMIZE_SIZE` to this option. + +When enabled, the LLM runner uses Hugging Face's parser-free inference pipeline +for `.tok` files while retaining the existing ExecuTorch `Tokenizer` interface. +A directory input uses `/tokenizer.tok`. JSON tokenizers continue +through the existing C++ `HFTokenizer`; all other tokenizer fallbacks are +unchanged. + +Create the artifact offline with the `tk-convert` tool from Hugging Face's +[`feat/tok-format`](https://github.com/huggingface/tokenizers/tree/feat/tok-format) +branch: + +```sh +cargo run --release --manifest-path tokenizers/Cargo.toml -p tk-convert -- \ + /path/to/tokenizer.json +``` + +This writes `/path/to/tokenizer.tok`. The v1 container supports BPE, Unigram, +WordPiece, and WordLevel models, but intentionally supports only the +normalizers and pre-tokenizers represented by the format. Conversion fails +instead of silently dropping unsupported behavior. + +This experiment currently supports host CMake builds. Android, Apple framework, +WASM, and Buck packaging still need explicit Rust target/toolchain integration. +The build requires Cargo and fetches the pinned Rust dependencies on its first +run. + +The Rust dependencies are pinned to Hugging Face tokenizers commit +`054bdf469b2cf416c0da953923d88c82f4d765b5` from `feat/tok-format`. Only +`tk-encode` and the zero-dependency `.tok` reader are linked; JSON conversion, +serde, training, and progress-bar code stay out of the runtime binary. The +Rust pipeline owns encoding. The C++ compatibility layer reads vocabulary and +special-token metadata from the same `.tok` image and applies byte-level +decoding when the format marks the model as byte-level. Other decoder chains +are not represented by `.tok` v1 and therefore retain raw-piece incremental +decode behavior. diff --git a/rust_tokenizer/binary-size.md b/rust_tokenizer/binary-size.md new file mode 100644 index 00000000..8ad6ab6e --- /dev/null +++ b/rust_tokenizer/binary-size.md @@ -0,0 +1,14 @@ +# Hugging Face `.tok` backend binary size + +Measured on Apple arm64 with Rust 1.98.1, `CMAKE_BUILD_TYPE=Release`, +`TOKENIZERS_OPTIMIZE_SIZE=ON`, dead stripping, and `gzip -9`. The smoke binary +loads a GPT-2 `.tok`, encodes `Hello world`, performs vocabulary lookups, and +decodes both output tokens. + +| Configuration | Stripped | Gzipped | +|---|---:|---:| +| Default (`TOKENIZERS_BUILD_HF_RUST_TOKENIZER=OFF`) | 0 B added | 0 B added | +| Opt-in `.tok` backend | 591,040 B | 298,961 B | + +The OFF configuration exposes no Rust CMake target and produces no Cargo build +directory. diff --git a/rust_tokenizer/src/lib.rs b/rust_tokenizer/src/lib.rs new file mode 100644 index 00000000..23c68431 --- /dev/null +++ b/rust_tokenizer/src/lib.rs @@ -0,0 +1,443 @@ +use std::ffi::{CStr, c_char, c_void}; +use std::panic::{AssertUnwindSafe, catch_unwind}; + +use tk_encode::pipeline::PipelineTokenizer; +use tk_serialization::{AddedEntry, Entry, TokFile, added_flag, kind}; + +struct Handle { + tokenizer: PipelineTokenizer, + // Keep the aligned `.tok` image alive. `PipelineTokenizer::from_tok` + // currently owns the structures it builds, but retaining the image makes + // the FFI safe if the upstream reader starts borrowing sections later. + file: TokFile, +} + +/// Create a tokenizer from a `.tok` path. +/// +/// # Safety +/// +/// `path` must be null or point to a valid NUL-terminated C string for the +/// duration of this call. A non-null return value must eventually be passed +/// exactly once to [`tokenizers_hf_destroy`]. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn tokenizers_hf_create(path: *const c_char) -> *mut c_void { + catch_unwind(AssertUnwindSafe(|| { + if path.is_null() { + return std::ptr::null_mut(); + } + let Ok(path) = unsafe { CStr::from_ptr(path) }.to_str() else { + return std::ptr::null_mut(); + }; + let Ok(file) = TokFile::open(path) else { + return std::ptr::null_mut(); + }; + let Ok(tokenizer) = PipelineTokenizer::from_tok(file.bytes()) else { + return std::ptr::null_mut(); + }; + Box::into_raw(Box::new(Handle { tokenizer, file })).cast() + })) + .unwrap_or(std::ptr::null_mut()) +} + +/// Encode one UTF-8 string into caller-owned token storage. +/// +/// # Safety +/// +/// `opaque` must be a live handle returned by +/// [`tokenizers_hf_create`]. `text` must address `text_len` readable +/// bytes unless `text_len` is zero. `output` must address `output_capacity` +/// writable `u32` values unless the capacity is zero. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn tokenizers_hf_encode( + opaque: *const c_void, + text: *const u8, + text_len: usize, + add_special_tokens: u8, + output: *mut u32, + output_capacity: usize, +) -> isize { + catch_unwind(AssertUnwindSafe(|| { + if opaque.is_null() || (text.is_null() && text_len != 0) { + return -1; + } + let handle = unsafe { &*opaque.cast::() }; + let bytes = if text_len == 0 { + &[] + } else { + unsafe { std::slice::from_raw_parts(text, text_len) } + }; + let Ok(text) = std::str::from_utf8(bytes) else { + return -1; + }; + + let Ok(tokens) = handle.tokenizer.encode(text, add_special_tokens != 0) else { + return -1; + }; + let Ok(token_count) = isize::try_from(tokens.len()) else { + return -1; + }; + if tokens.len() > output_capacity { + return token_count; + } + if !tokens.is_empty() && output.is_null() { + return -1; + } + for (index, token) in tokens.iter().enumerate() { + unsafe { output.add(index).write(token.id) }; + } + token_count + })) + .unwrap_or(-1) +} + +/// Return the number of vocabulary and added-token records in the `.tok`. +/// +/// This is a record count rather than a vocabulary size because an added token +/// can deliberately reuse a model-vocabulary id. The C++ compatibility layer +/// removes those duplicates while constructing its lookup maps. +/// +/// # Safety +/// +/// `opaque` must be null or a live handle returned by +/// [`tokenizers_hf_create`]. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn tokenizers_hf_token_count(opaque: *const c_void) -> isize { + catch_unwind(AssertUnwindSafe(|| { + if opaque.is_null() { + return -1; + } + let handle = unsafe { &*opaque.cast::() }; + let Ok(reader) = handle.file.reader() else { + return -1; + }; + let Ok(vocab) = reader.require::(kind::VOCAB_ENTRY) else { + return -1; + }; + let Ok(added) = reader.section::(kind::ADDED_ENTRY) else { + return -1; + }; + vocab + .len() + .checked_add(added.len()) + .and_then(|count| isize::try_from(count).ok()) + .unwrap_or(-1) + })) + .unwrap_or(-1) +} + +/// Read one vocabulary record. Returned string bytes borrow `opaque` and stay +/// valid until the handle is destroyed. +/// +/// # Safety +/// +/// `opaque` must be a live handle returned by +/// [`tokenizers_hf_create`]. Every output argument must point to +/// writable storage of its declared type. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn tokenizers_hf_token_at( + opaque: *const c_void, + index: usize, + id: *mut u32, + text: *mut *const u8, + text_len: *mut usize, + is_added: *mut u8, + is_special: *mut u8, +) -> i32 { + catch_unwind(AssertUnwindSafe(|| { + if opaque.is_null() + || id.is_null() + || text.is_null() + || text_len.is_null() + || is_added.is_null() + || is_special.is_null() + { + return -1; + } + let handle = unsafe { &*opaque.cast::() }; + let Ok(reader) = handle.file.reader() else { + return -1; + }; + let Ok(vocab) = reader.require::(kind::VOCAB_ENTRY) else { + return -1; + }; + let (slab, start, len, token_id, added, special) = if index < vocab.len() { + let Ok(slab) = reader.require::(kind::VOCAB_SLAB) else { + return -1; + }; + let entry = vocab[index]; + (slab, entry.start, entry.len, entry.id, false, false) + } else { + let Ok(entries) = reader.section::(kind::ADDED_ENTRY) else { + return -1; + }; + let Some(entry) = entries.get(index - vocab.len()).copied() else { + return -1; + }; + let Ok(slab) = reader.section::(kind::ADDED_SLAB) else { + return -1; + }; + ( + slab, + entry.start, + entry.len, + entry.id, + true, + entry.flags & added_flag::SPECIAL != 0, + ) + }; + let Some(end) = (start as usize).checked_add(len as usize) else { + return -1; + }; + let Some(bytes) = slab.get(start as usize..end) else { + return -1; + }; + if std::str::from_utf8(bytes).is_err() { + return -1; + } + unsafe { + id.write(token_id); + text.write(bytes.as_ptr()); + text_len.write(bytes.len()); + is_added.write(u8::from(added)); + is_special.write(u8::from(special)); + } + 0 + })) + .unwrap_or(-1) +} + +/// Return the first prefix token (`suffix == 0`) or last suffix token +/// (`suffix != 0`) from the `.tok` post-processor. Returns 0 when present, 1 +/// when the requested side is empty, and -1 on error. +/// +/// # Safety +/// +/// `opaque` must be a live handle returned by +/// [`tokenizers_hf_create`] and `token` must point to a writable +/// `u32`. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn tokenizers_hf_post_token( + opaque: *const c_void, + suffix: u8, + token: *mut u32, +) -> i32 { + catch_unwind(AssertUnwindSafe(|| { + if opaque.is_null() || token.is_null() { + return -1; + } + let handle = unsafe { &*opaque.cast::() }; + let Ok(reader) = handle.file.reader() else { + return -1; + }; + let Ok(tokens) = reader.section::(if suffix == 0 { + kind::POST_PREFIX + } else { + kind::POST_SUFFIX + }) else { + return -1; + }; + let value = if suffix == 0 { + tokens.first() + } else { + tokens.last() + }; + let Some(value) = value else { + return 1; + }; + unsafe { token.write(*value) }; + 0 + })) + .unwrap_or(-1) +} + +/// Return `.tok` config flags, or `u32::MAX` for an invalid handle/image. +/// +/// # Safety +/// +/// `opaque` must be null or a live handle returned by +/// [`tokenizers_hf_create`]. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn tokenizers_hf_config_flags(opaque: *const c_void) -> u32 { + catch_unwind(AssertUnwindSafe(|| { + if opaque.is_null() { + return u32::MAX; + } + let handle = unsafe { &*opaque.cast::() }; + handle + .file + .reader() + .map(|reader| reader.config.flags) + .unwrap_or(u32::MAX) + })) + .unwrap_or(u32::MAX) +} + +/// Destroy a tokenizer handle. +/// +/// # Safety +/// +/// `opaque` must be null or a live handle returned by +/// [`tokenizers_hf_create`] that has not previously been destroyed. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn tokenizers_hf_destroy(opaque: *mut c_void) { + if !opaque.is_null() { + let _ = catch_unwind(AssertUnwindSafe(|| { + drop(unsafe { Box::from_raw(opaque.cast::()) }); + })); + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::ffi::CString; + use tk_serialization::{Config, Writer, model, pretok, strings}; + + fn test_image() -> Vec { + let config = Config { + model: model::WORDLEVEL, + model_param: 0, + pretok: pretok::NONE, + pretok_param: 0, + flags: 0, + _pad0: 0, + added_first: [1 << (b'<' & 63), 0, 0, 0], + }; + let vocab_slab = b"hello"; + let vocab = [ + Entry { + start: 0, + len: 5, + id: 0, + }, + Entry { + start: 5, + len: 5, + id: 1, + }, + ]; + let added_slab = b""; + let added = [AddedEntry { + start: 0, + len: 3, + id: 2, + flags: added_flag::SPECIAL, + }]; + let mut model_strings = Vec::new(); + strings::push(&mut model_strings, ""); + strings::push(&mut model_strings, ""); + strings::push(&mut model_strings, ""); + + let mut writer = Writer::new(); + writer.push_one(kind::CONFIG, &config); + writer.push(kind::VOCAB_SLAB, vocab_slab); + writer.push(kind::VOCAB_ENTRY, &vocab); + writer.push(kind::ADDED_SLAB, added_slab); + writer.push(kind::ADDED_ENTRY, &added); + writer.push(kind::POST_PREFIX, &[2u32]); + writer.push(kind::POST_SUFFIX, &[2u32]); + writer.push(kind::MODEL_STRINGS, &model_strings); + writer.finish() + } + + fn with_handle(test: impl FnOnce(*mut c_void)) { + let path = std::env::temp_dir().join(format!( + "pytorch-tokenizers-hf-{}-{}.tok", + std::process::id(), + std::thread::current().name().unwrap_or("test") + )); + std::fs::write(&path, test_image()).unwrap(); + let c_path = CString::new(path.to_str().unwrap()).unwrap(); + let handle = unsafe { tokenizers_hf_create(c_path.as_ptr()) }; + assert!(!handle.is_null()); + test(handle); + unsafe { tokenizers_hf_destroy(handle) }; + std::fs::remove_file(path).unwrap(); + } + + #[test] + fn loads_and_encodes_tok() { + with_handle(|handle| { + let text = b"hello"; + let required = unsafe { + tokenizers_hf_encode( + handle, + text.as_ptr(), + text.len(), + 1, + std::ptr::null_mut(), + 0, + ) + }; + assert_eq!(required, 3); + let mut output = [0u32; 3]; + let written = unsafe { + tokenizers_hf_encode( + handle, + text.as_ptr(), + text.len(), + 1, + output.as_mut_ptr(), + output.len(), + ) + }; + assert_eq!(written, 3); + assert_eq!(output, [2, 1, 2]); + }); + } + + #[test] + fn exposes_tok_metadata() { + with_handle(|handle| { + assert_eq!(unsafe { tokenizers_hf_token_count(handle) }, 3); + + let mut id = 0; + let mut text = std::ptr::null(); + let mut len = 0; + let mut added = 0; + let mut special = 0; + assert_eq!( + unsafe { + tokenizers_hf_token_at( + handle, + 2, + &mut id, + &mut text, + &mut len, + &mut added, + &mut special, + ) + }, + 0 + ); + assert_eq!(id, 2); + assert_eq!(unsafe { std::slice::from_raw_parts(text, len) }, b""); + assert_eq!((added, special), (1, 1)); + + let mut token = 0; + assert_eq!( + unsafe { tokenizers_hf_post_token(handle, 0, &mut token) }, + 0 + ); + assert_eq!(token, 2); + assert_eq!( + unsafe { tokenizers_hf_post_token(handle, 1, &mut token) }, + 0 + ); + assert_eq!(token, 2); + assert_eq!(unsafe { tokenizers_hf_config_flags(handle) }, 0); + }); + } + + #[test] + fn rejects_non_tok_input() { + let path = std::env::temp_dir().join(format!( + "pytorch-tokenizers-hf-invalid-{}.tok", + std::process::id() + )); + std::fs::write(&path, b"not a tok").unwrap(); + let c_path = CString::new(path.to_str().unwrap()).unwrap(); + assert!(unsafe { tokenizers_hf_create(c_path.as_ptr()) }.is_null()); + std::fs::remove_file(path).unwrap(); + } +} diff --git a/src/rust_hf_tokenizer.cpp b/src/rust_hf_tokenizer.cpp new file mode 100644 index 00000000..7604eeb2 --- /dev/null +++ b/src/rust_hf_tokenizer.cpp @@ -0,0 +1,375 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +#include + +#include +#include +#include +#include +#include +#include + +namespace fs = std::filesystem; + +extern "C" { +void* tokenizers_hf_create(const char* path); +intptr_t tokenizers_hf_encode( + const void* handle, + const uint8_t* text, + size_t text_len, + uint8_t add_special_tokens, + uint32_t* output, + size_t output_capacity); +intptr_t tokenizers_hf_token_count(const void* handle); +int32_t tokenizers_hf_token_at( + const void* handle, + size_t index, + uint32_t* id, + const uint8_t** text, + size_t* text_len, + uint8_t* is_added, + uint8_t* is_special); +int32_t tokenizers_hf_post_token( + const void* handle, + uint8_t suffix, + uint32_t* token); +uint32_t tokenizers_hf_config_flags(const void* handle); +void tokenizers_hf_destroy(void* handle); +} + +namespace tokenizers { +namespace { + +// Mirrors tk_serialization::flag::BYTE_LEVEL. The `.tok` v1 format deliberately +// keeps these values stable as part of its on-disk schema. +constexpr uint32_t kByteLevelFlag = 1U << 2; + +std::optional byte_level_codepoint_to_byte(uint32_t codepoint) { + if ((codepoint >= 33 && codepoint <= 126) || + (codepoint >= 161 && codepoint <= 172) || + (codepoint >= 174 && codepoint <= 255)) { + return static_cast(codepoint); + } + if (codepoint < 256 || codepoint > 323) { + return std::nullopt; + } + const auto index = codepoint - 256; + if (index < 33) { + return static_cast(index); + } + if (index < 67) { + return static_cast(127 + index - 33); + } + return static_cast(173); +} + +std::string decode_byte_level(std::string_view piece) { + std::string decoded; + decoded.reserve(piece.size()); + for (size_t index = 0; index < piece.size();) { + const auto first = static_cast(piece[index]); + uint32_t codepoint = 0; + size_t length = 0; + if ((first & 0x80) == 0) { + codepoint = first; + length = 1; + } else if ((first & 0xE0) == 0xC0) { + codepoint = first & 0x1F; + length = 2; + } else if ((first & 0xF0) == 0xE0) { + codepoint = first & 0x0F; + length = 3; + } else if ((first & 0xF8) == 0xF0) { + codepoint = first & 0x07; + length = 4; + } else { + return std::string(piece); + } + if (length > piece.size() - index) { + return std::string(piece); + } + for (size_t offset = 1; offset < length; ++offset) { + const auto continuation = + static_cast(piece[index + offset]); + if ((continuation & 0xC0) != 0x80) { + return std::string(piece); + } + codepoint = (codepoint << 6) | (continuation & 0x3F); + } + const auto byte = byte_level_codepoint_to_byte(codepoint); + if (!byte) { + return std::string(piece); + } + decoded.push_back(static_cast(*byte)); + index += length; + } + return decoded; +} + +} // namespace + +RustHFTokenizer::RustHFTokenizer() : handle_(nullptr) {} + +RustHFTokenizer::~RustHFTokenizer() = default; + +void RustHFTokenizer::RustHandleDeleter::operator()(void* handle) const { + tokenizers_hf_destroy(handle); +} + +Error RustHFTokenizer::load(const std::string& path) { + initialized_ = false; + handle_.reset(); + token_map_.reset(); + added_token_map_.reset(); + special_token_ids_.clear(); + byte_level_ = false; + vocab_size_ = 0; + bos_tok_ = 0; + eos_tok_ = 0; + + std::string tokenizer_tok = path; + std::error_code fs_error; + if (fs::is_directory(path, fs_error)) { + const fs::path root(path); + tokenizer_tok = (root / "tokenizer.tok").string(); + } + if (fs_error || !fs::exists(tokenizer_tok, fs_error) || fs_error) { + return Error::LoadFailure; + } + + std::unique_ptr handle( + tokenizers_hf_create(tokenizer_tok.c_str())); + if (!handle) { + return Error::LoadFailure; + } + const auto metadata_error = load_metadata(handle.get()); + if (metadata_error != Error::Ok) { + return metadata_error; + } + + const auto flags = tokenizers_hf_config_flags(handle.get()); + if (flags == std::numeric_limits::max()) { + return Error::ParseFailure; + } + byte_level_ = (flags & kByteLevelFlag) != 0; + + handle_ = std::move(handle); + initialized_ = true; + return Error::Ok; +} + +Error RustHFTokenizer::load_metadata(const void* handle) { + const auto count = tokenizers_hf_token_count(handle); + if (count < 0 || + static_cast(count) > + static_cast(std::numeric_limits::max())) { + return Error::ParseFailure; + } + + struct TokenRecord { + std::string text; + uint64_t id; + bool added; + bool special; + }; + std::vector records; + records.reserve(static_cast(count)); + std::unordered_set added_ids; + for (size_t index = 0; index < static_cast(count); ++index) { + uint32_t id = 0; + const uint8_t* text = nullptr; + size_t text_len = 0; + uint8_t is_added = 0; + uint8_t is_special = 0; + if (tokenizers_hf_token_at( + handle, + index, + &id, + &text, + &text_len, + &is_added, + &is_special) != 0 || + (text == nullptr && text_len != 0)) { + return Error::ParseFailure; + } + records.push_back( + {std::string(reinterpret_cast(text), text_len), + id, + is_added != 0, + is_special != 0}); + if (is_added != 0) { + added_ids.insert(id); + if (is_special != 0) { + special_token_ids_.insert(id); + } + } + } + + std::vector> tokens; + std::vector> added_tokens; + std::vector bos_candidates; + std::vector eos_candidates; + tokens.reserve(records.size()); + added_tokens.reserve(added_ids.size()); + for (auto& record : records) { + if (record.special) { + if (record.text.find("bos") != std::string::npos || + record.text.find("begin") != std::string::npos) { + bos_candidates.push_back(record.id); + } + if (record.text.find("eos") != std::string::npos || + record.text.find("end") != std::string::npos) { + eos_candidates.push_back(record.id); + } + } + if (record.added) { + added_tokens.emplace_back(std::move(record.text), record.id); + } else if (added_ids.count(record.id) == 0) { + tokens.emplace_back(std::move(record.text), record.id); + } + } + auto added_map = TokenMap::create(added_tokens); + if (!added_map.ok()) { + return added_map.error(); + } + added_token_map_.emplace(std::move(*added_map)); + auto token_map = TokenMap::create(tokens); + if (!token_map.ok()) { + return token_map.error(); + } + token_map_.emplace(std::move(*token_map)); + vocab_size_ = static_cast( + token_map_->size() + added_token_map_->size()); + + uint32_t bos = 0; + uint32_t eos = 0; + const auto bos_status = + tokenizers_hf_post_token(handle, 0, &bos); + const auto eos_status = + tokenizers_hf_post_token(handle, 1, &eos); + if (bos_status < 0 || eos_status < 0) { + return Error::ParseFailure; + } + if (bos_status == 0) { + bos_tok_ = bos; + } + if (eos_status == 0) { + eos_tok_ = eos; + } + bool bos_found = bos_status == 0; + bool eos_found = eos_status == 0; + if (!bos_found || !eos_found) { + if (!bos_found && bos_candidates.size() == 1) { + bos_tok_ = bos_candidates.front(); + bos_found = true; + } + if (!eos_found && eos_candidates.size() == 1) { + eos_tok_ = eos_candidates.front(); + eos_found = true; + } + } + if (bos_found && !eos_found) { + eos_tok_ = bos_tok_; + } else if (!bos_found && eos_found) { + bos_tok_ = eos_tok_; + } + return Error::Ok; +} + +Result RustHFTokenizer::id_to_piece(uint64_t token) const { + if (!initialized_) { + return Error::Uninitialized; + } + if (auto piece = token_map_->tryGetString(token)) { + return std::string(*piece); + } + if (auto piece = added_token_map_->tryGetString(token)) { + return std::string(*piece); + } + return Error::OutOfRange; +} + +Result RustHFTokenizer::piece_to_id(const std::string& text) const { + if (!initialized_) { + return Error::Uninitialized; + } + if (auto id = token_map_->tryGetInteger(text)) { + return *id; + } + if (auto id = added_token_map_->tryGetInteger(text)) { + return *id; + } + return Error::OutOfRange; +} + +Result> RustHFTokenizer::encode( + const std::string& input, + int8_t bos, + int8_t eos) const { + if (!initialized_) { + return Error::Uninitialized; + } + if (input.size() > static_cast(std::numeric_limits::max())) { + return Error::EncodeFailure; + } + + std::vector output(input.size()); + auto count = tokenizers_hf_encode( + handle_.get(), + reinterpret_cast(input.data()), + input.size(), + static_cast(bos > 0 || eos > 0), + output.data(), + output.size()); + if (count < 0) { + return Error::EncodeFailure; + } + if (static_cast(count) > output.size()) { + output.resize(static_cast(count)); + count = tokenizers_hf_encode( + handle_.get(), + reinterpret_cast(input.data()), + input.size(), + static_cast(bos > 0 || eos > 0), + output.data(), + output.size()); + if (count < 0 || static_cast(count) > output.size()) { + return Error::EncodeFailure; + } + } + output.resize(static_cast(count)); + return std::vector(output.begin(), output.end()); +} + +Result RustHFTokenizer::decode( + uint64_t /*prev_token*/, + uint64_t token, + bool skip_special_tokens) const { + if (!initialized_) { + return Error::Uninitialized; + } + std::string_view piece; + if (auto regular = token_map_->tryGetString(token)) { + piece = *regular; + } else if (auto added = added_token_map_->tryGetString(token)) { + if (skip_special_tokens && special_token_ids_.count(token) != 0) { + return std::string(); + } + piece = *added; + } else { + return Error::DecodeFailure; + } + + if (!byte_level_) { + return std::string(piece); + } + return decode_byte_level(piece); +} + +} // namespace tokenizers