diff --git a/lib/llm/src/discovery/model.rs b/lib/llm/src/discovery/model.rs index a0565dc52f46..cb701eb498c9 100644 --- a/lib/llm/src/discovery/model.rs +++ b/lib/llm/src/discovery/model.rs @@ -15,6 +15,7 @@ use serde::Serialize; use super::ModelManagerError; use super::worker_monitor::LoadThresholdConfig; use super::worker_set::WorkerSet; +use crate::preprocessor::OpenAIPreprocessor; use crate::protocols::openai::ParsingOptions; use crate::types::{ @@ -689,6 +690,21 @@ impl Model { .ok_or_else(|| self.engine_error(self.has_chat_engine())) } + /// `None` when chat is served by a Python engine factory or the model has no Rust + /// tokenizer. + pub fn get_chat_preprocessor(&self) -> Option> { + self.select_worker_set_with(|ws| ws.chat_preprocessor.clone()) + } + + /// Either pipeline's preprocessor; both are built on the model's single tokenizer. + pub fn get_preprocessor(&self) -> Option> { + self.select_worker_set_with(|ws| { + ws.completions_preprocessor + .clone() + .or_else(|| ws.chat_preprocessor.clone()) + }) + } + pub fn get_completions_engine( &self, ) -> Result { diff --git a/lib/llm/src/discovery/model_manager.rs b/lib/llm/src/discovery/model_manager.rs index 701e070ee8bc..40d3682b6247 100644 --- a/lib/llm/src/discovery/model_manager.rs +++ b/lib/llm/src/discovery/model_manager.rs @@ -1,6 +1,7 @@ // SPDX-FileCopyrightText: Copyright (c) 2024-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +use crate::preprocessor::OpenAIPreprocessor; use std::{ collections::{HashMap, HashSet}, sync::{ @@ -700,6 +701,37 @@ impl ModelManager { .get_pooling_engine() } + pub fn get_chat_preprocessor(&self, model: &str) -> Option> { + self.models.get(model)?.get_chat_preprocessor() + } + + /// Whichever pipeline exists; both share the model's tokenizer. + pub fn get_preprocessor(&self, model: &str) -> Option> { + self.models.get(model)?.get_preprocessor() + } + + /// Attach preprocessors for an in-process model. Discovery-backed models get theirs + /// from the watcher as it builds their pipelines. + pub fn add_model_preprocessors( + &self, + model: &str, + card_checksum: &str, + chat: Option>, + completions: Option>, + ) -> Result<(), ModelManagerError> { + let model_entry = self.get_or_create_model(model); + let namespace = format!("__local_preprocessors_{}", model); + let mut ws = WorkerSet::new( + namespace.clone(), + card_checksum.to_string(), + Self::aggregated_local_card(), + ); + ws.chat_preprocessor = chat; + ws.completions_preprocessor = completions; + model_entry.add_worker_set(namespace, Arc::new(ws)); + Ok(()) + } + pub fn get_completions_engine( &self, model: &str, diff --git a/lib/llm/src/discovery/watcher.rs b/lib/llm/src/discovery/watcher.rs index 1e85dbc3f1e5..a53531bc0d43 100644 --- a/lib/llm/src/discovery/watcher.rs +++ b/lib/llm/src/discovery/watcher.rs @@ -1824,6 +1824,7 @@ impl ModelWatcher { let preprocessor = OpenAIPreprocessor::new_with_parts(card.clone(), formatter, tk.clone()) .context("OpenAIPreprocessor.new_with_parts")?; + worker_set.chat_preprocessor = Some(preprocessor.clone()); Some( routing .build_pipeline::< @@ -1866,6 +1867,7 @@ impl ModelWatcher { let preprocessor = OpenAIPreprocessor::new_with_parts(card.clone(), formatter, tk.clone()) .context("OpenAIPreprocessor::new_with_parts")?; + worker_set.completions_preprocessor = Some(preprocessor.clone()); let routing = preprocessed_routing.as_ref().ok_or_else(|| { anyhow::anyhow!("completions pipeline requires preprocessed routing") })?; diff --git a/lib/llm/src/discovery/worker_set.rs b/lib/llm/src/discovery/worker_set.rs index cabc7c0f57e3..5de8ecd34fff 100644 --- a/lib/llm/src/discovery/worker_set.rs +++ b/lib/llm/src/discovery/worker_set.rs @@ -17,6 +17,7 @@ use crate::{ discovery::{KvWorkerMonitor, allocator::AllocatorTrimOnDrop}, kv_router::{EncoderRouter, KvRouter, PrefillRouter}, model_card::ModelDeploymentCard, + preprocessor::OpenAIPreprocessor, types::{ RealtimeBidirectionalEngine, generic::tensor::TensorStreamingEngine, @@ -143,6 +144,11 @@ pub struct WorkerSet { card: ModelDeploymentCard, // Engines — each WorkerSet owns its own pipelines + /// Retained so callers can reuse this model's template and tokenizer instead of + /// building a second copy that can drift from the pipeline's. + pub(crate) chat_preprocessor: Option>, + pub(crate) completions_preprocessor: Option>, + pub(crate) chat_engine: Option, pub(crate) completions_engine: Option, pub(crate) embeddings_engine: Option, @@ -185,6 +191,8 @@ impl WorkerSet { endpoint_id: None, mdcsum, card, + chat_preprocessor: None, + completions_preprocessor: None, chat_engine: None, completions_engine: None, embeddings_engine: None, @@ -391,6 +399,8 @@ impl WorkerSet { endpoint_id: self.endpoint_id.clone(), mdcsum, card, + chat_preprocessor: self.chat_preprocessor.clone(), + completions_preprocessor: self.completions_preprocessor.clone(), chat_engine: lora_context_engine(&self.chat_engine, &lora_name), completions_engine: lora_context_engine(&self.completions_engine, &lora_name), embeddings_engine: lora_context_engine(&self.embeddings_engine, &lora_name), diff --git a/lib/llm/src/http/service.rs b/lib/llm/src/http/service.rs index 7984e3082e58..f3fb10d7a995 100644 --- a/lib/llm/src/http/service.rs +++ b/lib/llm/src/http/service.rs @@ -32,6 +32,7 @@ pub mod metrics; pub mod openapi_docs; pub mod realtime; pub mod service_v2; +pub mod tokenize; pub use axum; pub use frontend_extension::{ diff --git a/lib/llm/src/http/service/openapi_docs.rs b/lib/llm/src/http/service/openapi_docs.rs index 65f18bf4fb7f..38f8ebcbaa4b 100644 --- a/lib/llm/src/http/service/openapi_docs.rs +++ b/lib/llm/src/http/service/openapi_docs.rs @@ -318,6 +318,8 @@ fn generate_summary_for_path(path: &str) -> String { "/v1/embeddings" => "Create embeddings".to_string(), "/v1/responses" => "Create response".to_string(), "/v1/models" => "List available models".to_string(), + "/tokenize" => "Tokenize a prompt or chat conversation".to_string(), + "/detokenize" => "Detokenize a list of token ids".to_string(), "/health" => "Health check".to_string(), "/live" => "Liveness check".to_string(), "/metrics" => "Prometheus metrics".to_string(), @@ -353,6 +355,16 @@ fn generate_description_for_path(path: &str) -> String { "Lists the currently available models and provides basic information about each." .to_string() } + "/tokenize" => { + "Returns the token ids for a prompt (`prompt`) or a chat conversation (`messages`, \ + rendered through the model's chat template) without running inference. \ + Compatible with vLLM's tokenize API." + .to_string() + } + "/detokenize" => { + "Returns the text for a list of token ids. Compatible with vLLM's detokenize API." + .to_string() + } "/health" => { "Returns the health status of the service. Used for readiness probes." .to_string() diff --git a/lib/llm/src/http/service/service_v2.rs b/lib/llm/src/http/service/service_v2.rs index 14ebdd83873d..1a6fc76ff1fe 100644 --- a/lib/llm/src/http/service/service_v2.rs +++ b/lib/llm/src/http/service/service_v2.rs @@ -929,6 +929,10 @@ static HTTP_SVC_FILES_PATH_ENV: &str = "DYN_HTTP_SVC_FILES_PATH"; static HTTP_SVC_BATCHES_PATH_ENV: &str = "DYN_HTTP_SVC_BATCHES_PATH"; /// Environment variable to set the anthropic messages endpoint path (default: `/v1/messages`) static HTTP_SVC_ANTHROPIC_PATH_ENV: &str = "DYN_HTTP_SVC_ANTHROPIC_PATH"; +/// Environment variable to set the tokenize endpoint path (default: `/tokenize`) +static HTTP_SVC_TOKENIZE_PATH_ENV: &str = "DYN_HTTP_SVC_TOKENIZE_PATH"; +/// Environment variable to set the detokenize endpoint path (default: `/detokenize`) +static HTTP_SVC_DETOKENIZE_PATH_ENV: &str = "DYN_HTTP_SVC_DETOKENIZE_PATH"; /// Environment variable to enable the experimental vLLM-compatible /// `/inference/v1/generate` endpoint. Truthy value opts in; disabled by default. pub(super) static VLLM_ENABLE_INFERENCE_V1_GENERATE_ENV: &str = @@ -1102,6 +1106,11 @@ impl HttpServiceConfigBuilder { }, super::health::health_check_router(state.clone(), var(HTTP_SVC_HEALTH_PATH_ENV).ok()), super::health::live_check_router(state.clone(), var(HTTP_SVC_LIVE_PATH_ENV).ok()), + super::tokenize::tokenize_router(state.clone(), var(HTTP_SVC_TOKENIZE_PATH_ENV).ok()), + super::tokenize::detokenize_router( + state.clone(), + var(HTTP_SVC_DETOKENIZE_PATH_ENV).ok(), + ), ]; if admin_api_enabled { system_routes.push(super::busy_threshold::busy_threshold_router( diff --git a/lib/llm/src/http/service/tokenize.rs b/lib/llm/src/http/service/tokenize.rs new file mode 100644 index 000000000000..7847f5c39b9a --- /dev/null +++ b/lib/llm/src/http/service/tokenize.rs @@ -0,0 +1,297 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! `POST /tokenize` and `POST /detokenize` HTTP endpoints. +//! +//! `/tokenize` takes either `{ "prompt": ... }` or `{ "messages": [...] }`, the latter +//! rendering the model's chat template first. `/detokenize` is the inverse. +//! +//! Both run the model's own [`crate::preprocessor::OpenAIPreprocessor`] — the instance +//! `/v1/chat/completions` and `/v1/completions` run — so the reported count is the count +//! the model is sent. This module owns no template and no tokenizer of its own. + +use std::collections::HashMap; +use std::sync::Arc; + +use axum::{Json, Router, extract::State, extract::rejection::JsonRejection, routing::post}; +use serde::{Deserialize, Serialize}; + +use super::RouteDoc; +use super::openai::{ErrorMessage, ErrorResponse, check_ready}; +use super::service_v2; +use crate::model_card::ModelDeploymentCard; +use crate::protocols::openai::chat_completions::NvCreateChatCompletionRequest; + +#[derive(Debug, Clone, Deserialize)] +pub struct TokenizeCompletionRequest { + pub model: Option, + pub prompt: String, + /// Defaults to `false`: `/v1/completions` applies no template and tokenizes without + /// special tokens, so the prompt reaches the engine with no BOS. An explicit `true` + /// returns 501 rather than reporting a count this deployment never produces. + #[serde(default)] + pub add_special_tokens: bool, + #[serde(default)] + pub return_token_strs: Option, +} + +#[derive(Debug, Clone, Deserialize)] +pub struct TokenizeChatRequest { + pub model: Option, + pub messages: Vec, + #[serde(default = "default_true")] + pub add_generation_prompt: bool, + #[serde(default)] + pub continue_final_message: bool, + /// Chat templates already insert the model's special tokens; an explicit `true` + /// returns 501 rather than double-inserting them. + #[serde(default)] + pub add_special_tokens: bool, + pub chat_template: Option, + pub chat_template_kwargs: Option>, + pub tools: Option>, + #[serde(default)] + pub return_token_strs: Option, + /// Accepted for schema compatibility; both only affect multimodal preprocessing, + /// which this endpoint does not perform. + pub media_io_kwargs: Option, + pub mm_processor_kwargs: Option, +} + +#[derive(Debug, Deserialize)] +#[serde(untagged)] +pub enum TokenizeRequest { + /// Boxed: the chat form is ~5x the size of the completion form, and clippy's + /// `large_enum_variant` would otherwise make every request pay for the bigger one. + Chat(Box), + Completion(TokenizeCompletionRequest), +} + +#[derive(Debug, Serialize)] +pub struct TokenizeResponse { + pub count: usize, + pub max_model_len: u32, + pub tokens: Vec, + /// Always serialized, `null` when not requested. Decoded text rather than vocabulary + /// spellings; see [`crate::preprocessor::OpenAIPreprocessor::token_strings`]. + pub token_strs: Option>, +} + +#[derive(Debug, Clone, Deserialize)] +pub struct DetokenizeRequest { + pub model: Option, + pub tokens: Vec, +} + +#[derive(Debug, Serialize)] +pub struct DetokenizeResponse { + pub prompt: String, +} + +fn default_true() -> bool { + true +} + +/// Turn an extractor rejection into the same error shape the rest of the API uses. +/// Worth spelling out for `/tokenize`: a body matching neither arm of the untagged +/// union otherwise surfaces as a bare `data did not match any variant` 422. +fn invalid_request(rejection: JsonRejection, expected: &str) -> ErrorResponse { + bad_request(format!("{rejection}. Expected {expected}.")) +} + +fn bad_request(message: String) -> ErrorResponse { + ErrorMessage::from_http_error(super::error::HttpError { code: 400, message }) +} + +/// Resolve the model a request targets. A body without `model` is honored only when one +/// model is registered. +fn resolve_model( + state: &Arc, + requested: Option<&str>, +) -> Result { + let mut cards = state.manager().get_model_cards(); + match requested { + Some(name) => cards + .into_iter() + .find(|card| card.display_name == name) + .ok_or_else(ErrorMessage::model_not_found), + None if cards.len() == 1 => Ok(cards.remove(0)), + None => Err(ErrorMessage::model_not_found()), + } +} + +fn no_preprocessor(model: &str) -> ErrorResponse { + ErrorMessage::not_implemented_error(format!( + "Model '{model}' has no Rust tokenizer pipeline, so it cannot be tokenized here" + )) +} + +fn tokenizer_failed(what: &str, error: impl std::fmt::Display) -> ErrorResponse { + ErrorMessage::internal_server_error(&format!("Tokenizer {what} failed: {error}")) +} + +/// Multimodal parts expand to many more tokens at inference than the single placeholder +/// a chat template emits, and this endpoint does no media preprocessing — so a count for +/// such a request would be wrong rather than merely approximate. Detect and refuse. +fn first_non_text_part(messages: &[serde_json::Value]) -> Option<&str> { + messages + .iter() + .filter_map(|message| message.get("content")?.as_array()) + .flatten() + .filter_map(|part| part.get("type")?.as_str()) + .find(|kind| *kind != "text") +} + +/// Build the request the generate path would have received. Deserializing into the real +/// request type is deliberate: it validates the messages the same way +/// `/v1/chat/completions` does, so a body this endpoint accepts is one the model +/// would accept. +fn chat_request( + req: &TokenizeChatRequest, + model: &str, +) -> Result { + serde_json::from_value(serde_json::json!({ + "model": model, + "messages": req.messages, + "tools": req.tools, + "chat_template_kwargs": req.chat_template_kwargs, + })) + .map_err(|e| bad_request(format!("Invalid chat request: {e}"))) +} + +async fn tokenize( + State(state): State>, + request: Result, JsonRejection>, +) -> Result, ErrorResponse> { + check_ready(&state)?; + let Json(request) = request.map_err(|rejection| { + invalid_request( + rejection, + "either `prompt` (completion form) or `messages` (chat form)", + ) + })?; + + let (card, preprocessor, encoding, return_token_strs) = match request { + TokenizeRequest::Completion(req) => { + // Before the lookup, so an unsupported flag reads the same on a known and an + // unknown model. + if req.add_special_tokens { + return Err(ErrorMessage::not_implemented_error( + "`add_special_tokens` is not supported: this model's /v1/completions \ + tokenizes without them, and /tokenize reports what it would send", + )); + } + let card = resolve_model(&state, req.model.as_deref())?; + let preprocessor = state + .manager() + .get_preprocessor(&card.display_name) + .ok_or_else(|| no_preprocessor(&card.display_name))?; + let encoding = preprocessor + .tokenize_completion(&req.prompt) + .await + .map_err(|e| tokenizer_failed("encode", e))?; + (card, preprocessor, encoding, req.return_token_strs) + } + TokenizeRequest::Chat(req) => { + // Rather than return a count for a prompt we did not build. + if req.continue_final_message { + return Err(ErrorMessage::not_implemented_error( + "`continue_final_message` is not yet supported", + )); + } + if req.chat_template.is_some() { + return Err(ErrorMessage::not_implemented_error( + "Per-request `chat_template` override is not yet supported", + )); + } + if req.media_io_kwargs.is_some() || req.mm_processor_kwargs.is_some() { + return Err(ErrorMessage::not_implemented_error( + "`media_io_kwargs` and `mm_processor_kwargs` are not yet supported", + )); + } + if let Some(kind) = first_non_text_part(&req.messages) { + return Err(ErrorMessage::not_implemented_error(format!( + "Multimodal content is not yet supported by /tokenize (message part type '{kind}')" + ))); + } + if req.add_special_tokens { + return Err(ErrorMessage::not_implemented_error( + "`add_special_tokens` on the chat form is not yet supported; the chat \ + template already carries the model's special tokens", + )); + } + + let card = resolve_model(&state, req.model.as_deref())?; + let preprocessor = state + .manager() + .get_chat_preprocessor(&card.display_name) + .ok_or_else(|| { + ErrorMessage::not_implemented_error(format!( + "Model '{}' has no Rust chat pipeline to render a chat template with", + card.display_name + )) + })?; + let mut chat = chat_request(&req, &card.display_name)?; + let encoding = preprocessor + .tokenize_chat(&mut chat, req.add_generation_prompt) + .await + // Covers rendering and encoding; the status follows rendering, the half a + // caller can act on. One entry point costs an encode failure the same 400. + .map_err(|e| bad_request(format!("Failed to tokenize chat request: {e}")))?; + (card, preprocessor, encoding, req.return_token_strs) + } + }; + + let tokens = encoding.token_ids().to_vec(); + let token_strs = return_token_strs + .unwrap_or(false) + .then(|| preprocessor.token_strings(&tokens)) + .transpose() + .map_err(|e| tokenizer_failed("decode", e))?; + + Ok(Json(TokenizeResponse { + count: tokens.len(), + max_model_len: card.effective_context_length(), + tokens, + token_strs, + })) +} + +async fn detokenize( + State(state): State>, + request: Result, JsonRejection>, +) -> Result, ErrorResponse> { + check_ready(&state)?; + let Json(request) = + request.map_err(|rejection| invalid_request(rejection, "`tokens`, a list of token ids"))?; + let card = resolve_model(&state, request.model.as_deref())?; + let prompt = state + .manager() + .get_preprocessor(&card.display_name) + .ok_or_else(|| no_preprocessor(&card.display_name))? + .detokenize(&request.tokens, false) + .map_err(|e| tokenizer_failed("decode", e))?; + Ok(Json(DetokenizeResponse { prompt })) +} + +pub fn tokenize_router( + state: Arc, + path: Option, +) -> (Vec, Router) { + let path = path.unwrap_or_else(|| "/tokenize".to_string()); + let doc = RouteDoc::new(axum::http::Method::POST, &path); + let router = Router::new().route(&path, post(tokenize)).with_state(state); + (vec![doc], router) +} + +pub fn detokenize_router( + state: Arc, + path: Option, +) -> (Vec, Router) { + let path = path.unwrap_or_else(|| "/detokenize".to_string()); + let doc = RouteDoc::new(axum::http::Method::POST, &path); + let router = Router::new() + .route(&path, post(detokenize)) + .with_state(state); + (vec![doc], router) +} diff --git a/lib/llm/src/preprocessor.rs b/lib/llm/src/preprocessor.rs index 216072153358..6c40bbbe868a 100644 --- a/lib/llm/src/preprocessor.rs +++ b/lib/llm/src/preprocessor.rs @@ -421,6 +421,98 @@ struct PreprocessRequestOptions { preserve_omitted_max_tokens: bool, } +/// Renders `req` with `add_generation_prompt` forced to a given value. +/// +/// MiniJinja's `context!{ ..a, ..b }` resolves `a` first, so `chat_template_args` can add +/// keys but cannot override the flag the renderer already put in the context. +struct GenerationPromptOverride<'a, R>(&'a R, bool); + +impl OAIChatLikeRequest for GenerationPromptOverride<'_, R> { + fn model(&self) -> String { + self.0.model() + } + + fn messages(&self) -> minijinja::value::Value { + self.0.messages() + } + + fn typed_messages(&self) -> Option<&[dynamo_protocols::types::ChatCompletionRequestMessage]> { + self.0.typed_messages() + } + + fn tools(&self) -> Option { + self.0.tools() + } + + fn tool_choice(&self) -> Option { + self.0.tool_choice() + } + + fn response_format(&self) -> Option { + self.0.response_format() + } + + fn should_add_generation_prompt(&self) -> bool { + self.1 + } + + fn prompt_input_type(&self) -> PromptInput { + self.0.prompt_input_type() + } + + fn extract_tokens(&self) -> Option { + self.0.extract_tokens() + } + + fn extract_text(&self) -> Option { + self.0.extract_text() + } + + fn chat_template_args(&self) -> Option<&std::collections::HashMap> { + self.0.chat_template_args() + } + + fn mm_processor_kwargs(&self) -> Option<&serde_json::Value> { + self.0.mm_processor_kwargs() + } +} + +impl AnnotationsProvider for GenerationPromptOverride<'_, R> { + fn annotations(&self) -> Option> { + self.0.annotations() + } +} + +impl SamplingOptionsProvider for GenerationPromptOverride<'_, R> { + fn extract_sampling_options( + &self, + ) -> anyhow::Result { + self.0.extract_sampling_options() + } +} + +impl StopConditionsProvider for GenerationPromptOverride<'_, R> { + fn extract_stop_conditions(&self) -> anyhow::Result { + self.0.extract_stop_conditions() + } +} + +impl OutputOptionsProvider for GenerationPromptOverride<'_, R> { + fn extract_output_options(&self) -> anyhow::Result { + self.0.extract_output_options() + } +} + +impl NvExtProvider for GenerationPromptOverride<'_, R> { + fn nvext(&self) -> Option<&crate::protocols::common::extensions::NvExt> { + self.0.nvext() + } + + fn raw_prompt(&self) -> Option { + self.0.raw_prompt() + } +} + pub struct OpenAIPreprocessor { mdcsum: String, formatter: Arc, @@ -1040,6 +1132,58 @@ impl OpenAIPreprocessor { self.tokenizer.encode(s) } + /// Every normalization [`Self::generate`] applies to a chat request before templating. + /// + /// These mutate `chat_template_args` and `tool_choice`, so templating without them + /// renders a different prompt. A step added here reaches every caller. + pub(crate) fn normalize_chat_request(&self, request: &mut NvCreateChatCompletionRequest) { + self.apply_default_thinking_mode(request); + Self::normalize_thinking_arg(request, self.runtime_config.reasoning_parser.as_deref()); + } + + /// Normalize, template and encode a chat request as [`Self::generate`] does, stopping + /// before the worker hop. + pub async fn tokenize_chat( + &self, + request: &mut NvCreateChatCompletionRequest, + add_generation_prompt: bool, + ) -> anyhow::Result { + self.normalize_chat_request(request); + let rendered = self + .apply_template(&GenerationPromptOverride(&*request, add_generation_prompt))? + .ok_or_else(|| anyhow::anyhow!("chat template produced no prompt"))?; + self.tokenize_completion(&rendered).await + } + + /// Encode a bare prompt as `/v1/completions` does: no template, no special tokens. + pub async fn tokenize_completion(&self, prompt: &str) -> anyhow::Result { + Self::encode_text(self.tokenizer.clone(), prompt).await + } + + /// Decode token ids through this model's tokenizer. + pub fn detokenize( + &self, + token_ids: &[TokenIdType], + skip_special_tokens: bool, + ) -> anyhow::Result { + Ok(self + .tokenizer + .decode(token_ids, skip_special_tokens)? + .into()) + } + + /// The text each token contributes, decoded per id. + /// + /// Not the vocabulary spelling: that is only reachable from an `Encoding::Hf`, and the + /// prefix cache normalizes every encode to `Encoding::Sp`, so it would vary with + /// `DYN_TOKENIZER_CACHE`. A partial multi-byte token decodes to U+FFFD. + pub fn token_strings(&self, token_ids: &[TokenIdType]) -> anyhow::Result> { + token_ids + .iter() + .map(|id| self.detokenize(&[*id], false)) + .collect() + } + /// Translate a [`NvCreateChatCompletionRequest`] request to a common completion request. /// Returns the common completion request, a hashmap of annotations, and a boolean /// indicating whether the rendered prompt ends with a reasoning start token (e.g., @@ -2350,12 +2494,11 @@ impl OpenAIPreprocessor { Ok(()) } - async fn encode_with_timing( - &self, + /// Encode `prompt`, stripping NUL bytes and offloading to the blocking pool. + pub async fn encode_text( + tokenizer: Arc, prompt: &str, - tracker: Option<&RequestTracker>, ) -> anyhow::Result { - let encode_start = Instant::now(); // Offload the CPU-heavy BPE encode to the bounded blocking pool instead of running it on // the async event loop. For long prompts at high concurrency, a synchronous encode here // stalls the frontend tokio runtime for seconds, starving the I/O tasks that share the @@ -2367,8 +2510,16 @@ impl OpenAIPreprocessor { } else { prompt.to_string() }; - let tokenizer = self.tokenizer.clone(); - let encoding = tokio::task::spawn_blocking(move || tokenizer.encode(&owned)).await??; + tokio::task::spawn_blocking(move || tokenizer.encode(&owned)).await? + } + + async fn encode_with_timing( + &self, + prompt: &str, + tracker: Option<&RequestTracker>, + ) -> anyhow::Result { + let encode_start = Instant::now(); + let encoding = Self::encode_text(self.tokenizer.clone(), prompt).await?; if let Some(t) = tracker { t.record_tokenize_latency(encode_start.elapsed()); } @@ -3723,11 +3874,7 @@ impl // Apply the deployment default before parser-specific normalization so // it can override an implicit model default (for example Kimi K2.5), // while explicit request controls still take precedence. - self.apply_default_thinking_mode(&mut request); - Self::normalize_thinking_arg( - &mut request, - self.runtime_config.reasoning_parser.as_deref(), - ); + self.normalize_chat_request(&mut request); // create a response generator let response_generator = request.response_generator(context.id().to_string()); @@ -5732,4 +5879,159 @@ mod tests { "s3:// query params identify objects and must not collide" ); } + /// `tokenize_chat` and the generate path must produce the same token ids. + /// + /// The request sends only `thinking`, so the `enable_thinking` the template branches on + /// exists only if normalization derived it; the second assertion checks it did, which + /// keeps this from passing when neither path normalized. + #[tokio::test] + async fn tokenize_chat_matches_the_generate_path_token_ids() { + use crate::model_card::ModelDeploymentCard; + use crate::preprocessor::prompt::prompt_formatter_from_mdc; + use dynamo_renderer::PromptFormatter; + use std::io::Write; + + let mut template = tempfile::Builder::new() + .suffix(".jinja") + .tempfile() + .expect("tempfile"); + template + .write_all( + b"{% for m in messages %}{{ m['content'] }}{% endfor %}\ + {% if enable_thinking %} thinking thinking thinking{% endif %}", + ) + .expect("write template"); + let template = template.into_temp_path(); + + let card = ModelDeploymentCard::load_from_disk( + "tests/data/sample-models/TinyLlama_v1.1", + Some(template.as_ref()), + ) + .expect("load card"); + let PromptFormatter::OAI(formatter) = prompt_formatter_from_mdc(&card).expect("formatter"); + let preprocessor = OpenAIPreprocessor::new_with_parts( + card.clone(), + formatter, + card.tokenizer().expect("tokenizer"), + ) + .expect("preprocessor"); + + let body = serde_json::json!({ + "model": card.display_name, + "messages": [{"role": "user", "content": "What is 2+2?"}], + "chat_template_kwargs": {"thinking": true}, + }); + + let mut via_generate: NvCreateChatCompletionRequest = + serde_json::from_value(body.clone()).expect("request"); + preprocessor.normalize_chat_request(&mut via_generate); + let (preprocessed, _, _) = preprocessor + .preprocess_request(&via_generate, None) + .await + .expect("preprocess"); + + let mut via_tokenize: NvCreateChatCompletionRequest = + serde_json::from_value(body).expect("request"); + let encoding = preprocessor + .tokenize_chat(&mut via_tokenize, true) + .await + .expect("tokenize_chat"); + + assert_eq!( + encoding.token_ids(), + preprocessed.token_ids.as_slice(), + "/tokenize and the generate path must agree on the token ids" + ); + + let mut without: NvCreateChatCompletionRequest = + serde_json::from_value(serde_json::json!({ + "model": card.display_name, + "messages": [{"role": "user", "content": "What is 2+2?"}], + })) + .expect("request"); + let baseline = preprocessor + .tokenize_chat(&mut without, true) + .await + .expect("tokenize_chat"); + assert!( + encoding.token_ids().len() > baseline.token_ids().len(), + "the thinking alias must reach the template via normalization, else this guard is vacuous" + ); + } + /// Null bytes are stripped before encoding, so a prompt containing one tokenizes the + /// same as a prompt without. + #[tokio::test] + async fn tokenize_completion_strips_null_bytes_like_the_generate_path() { + use crate::model_card::ModelDeploymentCard; + use crate::preprocessor::prompt::prompt_formatter_from_mdc; + use dynamo_renderer::PromptFormatter; + + let card = + ModelDeploymentCard::load_from_disk("tests/data/sample-models/TinyLlama_v1.1", None) + .expect("load card"); + let PromptFormatter::OAI(formatter) = PromptFormatter::no_op(); + let _ = prompt_formatter_from_mdc(&card); + let preprocessor = OpenAIPreprocessor::new_with_parts( + card.clone(), + formatter, + card.tokenizer().expect("tokenizer"), + ) + .expect("preprocessor"); + + let with_null = preprocessor + .tokenize_completion("Hello\0, world!") + .await + .expect("encode"); + let without_null = preprocessor + .tokenize_completion("Hello, world!") + .await + .expect("encode"); + assert_eq!( + with_null.token_ids(), + without_null.token_ids(), + "null bytes must be stripped before encoding, as the generate path does" + ); + } + + /// `tokenize_completion` must equal what the `/v1/completions` pipeline sends the + /// worker. Pinned because it is short enough to look replaceable by any encode call. + #[tokio::test] + async fn tokenize_completion_matches_the_completions_pipeline() { + use crate::model_card::ModelDeploymentCard; + use crate::protocols::openai::completions::NvCreateCompletionRequest; + use dynamo_renderer::PromptFormatter; + + let card = + ModelDeploymentCard::load_from_disk("tests/data/sample-models/TinyLlama_v1.1", None) + .expect("load card"); + // Exactly how the watcher builds the completions pipeline: a no-op formatter. + let PromptFormatter::OAI(no_op) = PromptFormatter::no_op(); + let preprocessor = OpenAIPreprocessor::new_with_parts( + card.clone(), + no_op, + card.tokenizer().expect("tokenizer"), + ) + .expect("preprocessor"); + + let request: NvCreateCompletionRequest = serde_json::from_value(serde_json::json!({ + "model": card.display_name, + "prompt": "Hello, world!", + })) + .expect("request"); + + let (preprocessed, _, _) = preprocessor + .preprocess_request(&request, None) + .await + .expect("preprocess"); + let via_tokenize = preprocessor + .tokenize_completion("Hello, world!") + .await + .expect("tokenize_completion"); + + assert_eq!( + via_tokenize.token_ids(), + preprocessed.token_ids.as_slice(), + "/tokenize must reproduce the /v1/completions token ids exactly" + ); + } } diff --git a/lib/llm/tests/http-tokenize.rs b/lib/llm/tests/http-tokenize.rs new file mode 100644 index 000000000000..52e8d3d45762 --- /dev/null +++ b/lib/llm/tests/http-tokenize.rs @@ -0,0 +1,582 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Integration tests for `POST /tokenize` and `POST /detokenize`. +//! +//! Boots an `HttpService` with a sample-model card registered on the `ModelManager`. +//! No engine is needed: both endpoints only read the card and its tokenizer file. + +use dynamo_llm::{ + http::service::service_v2::HttpService, + model_card::ModelDeploymentCard, + preprocessor::{OpenAIPreprocessor, prompt::prompt_formatter_from_mdc}, +}; +use dynamo_renderer::PromptFormatter; +use dynamo_runtime::CancellationToken; +use serde_json::{Value, json}; + +#[path = "common/ports.rs"] +mod ports; +use ports::bind_random_port; + +/// No chat template; `max_position_embeddings` is 2048 and BOS is token id 1. +const FIXTURE_COMPLETION: &str = "tests/data/sample-models/TinyLlama_v1.1"; +const MODEL_COMPLETION: &str = "tinyllama"; + +/// Carries a chat template in `tokenizer_config.json`. +const FIXTURE_CHAT: &str = "tests/data/sample-models/mock-llama-3.1-8b-instruct"; +const MODEL_CHAT: &str = "llama3-chat"; + +/// tiktoken tokenizer — exercises the non-HuggingFace encoding path. +const FIXTURE_TIKTOKEN: &str = "tests/data/sample-models/mock-tiktoken"; +const MODEL_TIKTOKEN: &str = "tiktoken-model"; + +struct Service { + /// Kept alive for the service's lifetime: the chat template is read lazily at + /// render time, so dropping the temp file early makes /tokenize 501. + _chat_template: Option, + port: u16, + cancel: CancellationToken, + join: tokio::task::JoinHandle>, +} + +impl Service { + async fn start(fixture: &str, display_name: &str) -> Self { + Self::start_with(fixture, display_name, None, |_| {}).await + } + + async fn start_with( + fixture: &str, + display_name: &str, + chat_template: Option, + tweak: impl FnOnce(&mut ModelDeploymentCard), + ) -> Self { + let (listener, port) = bind_random_port().await; + let service = HttpService::builder() + .port(port) + .host("127.0.0.1") + .build() + .expect("failed to build HTTP service"); + + let mut card = ModelDeploymentCard::load_from_disk(fixture, chat_template.as_deref()) + .expect("load_from_disk"); + card.display_name = display_name.to_string(); + tweak(&mut card); + + // What the discovery watcher builds for a real model. Registering only the card + // would not exercise the objects production uses. + let tokenizer = card.tokenizer().expect("tokenizer"); + let chat = prompt_formatter_from_mdc(&card).ok().map(|formatter| { + let PromptFormatter::OAI(formatter) = formatter; + OpenAIPreprocessor::new_with_parts(card.clone(), formatter, tokenizer.clone()) + .expect("chat preprocessor") + }); + let PromptFormatter::OAI(no_op) = PromptFormatter::no_op(); + let completions = + OpenAIPreprocessor::new_with_parts(card.clone(), no_op, tokenizer.clone()) + .expect("completions preprocessor"); + + let checksum = card.mdcsum().to_string(); + let manager = service.model_manager(); + manager + .save_model_card("test-instance-key", card) + .expect("save_model_card"); + manager + .add_model_preprocessors(display_name, &checksum, chat, Some(completions)) + .expect("add_model_preprocessors"); + + let cancel = CancellationToken::new(); + let join = service.spawn_with_listener(cancel.clone(), listener).await; + Self { + _chat_template: chat_template, + port, + cancel, + join, + } + } + + async fn post(&self, path: &str, body: Value) -> reqwest::Response { + reqwest::Client::builder() + .no_proxy() + .build() + .expect("client") + .post(format!("http://127.0.0.1:{}{path}", self.port)) + .json(&body) + .send() + .await + .expect("request send failed") + } + + /// POST expecting 200, returning the decoded body. + async fn post_ok(&self, path: &str, body: Value) -> Value { + let resp = self.post(path, body).await; + let status = resp.status(); + let body: Value = resp.json().await.expect("json body"); + assert_eq!(status, 200, "body: {body}"); + body + } + + async fn shutdown(self) { + self.cancel.cancel(); + let _ = self.join.await; + } +} + +fn tokens(body: &Value) -> Vec { + body["tokens"] + .as_array() + .expect("tokens array") + .iter() + .map(|t| t.as_u64().expect("token id")) + .collect() +} + +#[tokio::test] +async fn tokenize_completion_matches_dynamo_not_vllm_defaults() { + let svc = Service::start(FIXTURE_COMPLETION, MODEL_COMPLETION).await; + + let body = svc + .post_ok( + "/tokenize", + json!({"model": MODEL_COMPLETION, "prompt": "Hello, world!"}), + ) + .await; + + let tokens = tokens(&body); + assert_eq!(body["count"].as_u64().unwrap() as usize, tokens.len()); + assert_eq!(body["max_model_len"].as_u64(), Some(2048)); + // Dynamo's /v1/completions adds no BOS, so neither does the count reported for it. + assert_ne!(tokens.first(), Some(&1), "TinyLlama BOS must not be added"); + assert!(body.get("token_strs").is_none_or(Value::is_null)); + + svc.shutdown().await; +} + +#[tokio::test] +async fn tokenize_completion_rejects_add_special_tokens() { + let svc = Service::start(FIXTURE_COMPLETION, MODEL_COMPLETION).await; + + let resp = svc + .post( + "/tokenize", + json!({"model": MODEL_COMPLETION, "prompt": "hi", "add_special_tokens": true}), + ) + .await; + assert_eq!(resp.status(), 501); + + svc.shutdown().await; +} + +#[tokio::test] +async fn tokenize_returns_token_strs() { + let svc = Service::start(FIXTURE_COMPLETION, MODEL_COMPLETION).await; + + let body = svc + .post_ok( + "/tokenize", + json!({"model": MODEL_COMPLETION, "prompt": "Hello", "return_token_strs": true}), + ) + .await; + + let strs = body["token_strs"].as_array().expect("token_strs present"); + assert_eq!(tokens(&body).len(), strs.len()); + assert!(strs.iter().all(Value::is_string)); + // Decoded text, not the byte-level vocabulary spelling vLLM returns. + assert_eq!(strs.first().and_then(Value::as_str), Some("Hello")); + + svc.shutdown().await; +} + +#[tokio::test] +async fn tokenize_tiktoken_model() { + let svc = Service::start(FIXTURE_TIKTOKEN, MODEL_TIKTOKEN).await; + + let body = svc + .post_ok( + "/tokenize", + json!({"model": MODEL_TIKTOKEN, "prompt": "Hello, world!", "return_token_strs": true}), + ) + .await; + + let tokens = tokens(&body); + assert!(!tokens.is_empty()); + assert_eq!( + body["token_strs"].as_array().expect("token_strs").len(), + tokens.len(), + "the tiktoken path decodes each id to build token_strs" + ); + + svc.shutdown().await; +} + +#[tokio::test] +async fn tokenize_chat_form_applies_template() { + let svc = Service::start(FIXTURE_CHAT, MODEL_CHAT).await; + + let chat = svc + .post_ok( + "/tokenize", + json!({ + "model": MODEL_CHAT, + "messages": [{"role": "user", "content": "Hello"}], + }), + ) + .await; + let bare = svc + .post_ok( + "/tokenize", + json!({"model": MODEL_CHAT, "prompt": "Hello", "add_special_tokens": false}), + ) + .await; + + assert!( + tokens(&chat).len() > tokens(&bare).len(), + "chat form should add the template's tokens on top of the bare prompt" + ); + + svc.shutdown().await; +} + +#[tokio::test] +async fn tokenize_chat_honors_add_generation_prompt() { + let svc = Service::start(FIXTURE_CHAT, MODEL_CHAT).await; + + let body = json!({"model": MODEL_CHAT, "messages": [{"role": "user", "content": "Hello"}]}); + let with = svc.post_ok("/tokenize", body.clone()).await; + + let mut without = body; + without["add_generation_prompt"] = json!(false); + let without = svc.post_ok("/tokenize", without).await; + + assert!( + tokens(&with).len() > tokens(&without).len(), + "add_generation_prompt=true must append the assistant header" + ); + + svc.shutdown().await; +} + +#[tokio::test] +async fn tokenize_chat_rejects_unsupported_fields() { + let svc = Service::start(FIXTURE_CHAT, MODEL_CHAT).await; + + for extra in [ + json!({"continue_final_message": true}), + json!({"chat_template": "{{ messages[0].content }}"}), + json!({"mm_processor_kwargs": {"foo": 1}}), + json!({"media_io_kwargs": {"image": {}}}), + ] { + let mut body = + json!({"model": MODEL_CHAT, "messages": [{"role": "user", "content": "hi"}]}); + for (k, v) in extra.as_object().unwrap() { + body[k] = v.clone(); + } + let resp = svc.post("/tokenize", body).await; + assert_eq!(resp.status(), 501, "unsupported field {extra} must not 200"); + } + + svc.shutdown().await; +} + +#[tokio::test] +async fn tokenize_chat_without_template_is_not_implemented() { + let svc = Service::start(FIXTURE_COMPLETION, MODEL_COMPLETION).await; + + let resp = svc + .post( + "/tokenize", + json!({ + "model": MODEL_COMPLETION, + "messages": [{"role": "user", "content": "hi"}], + }), + ) + .await; + assert_eq!(resp.status(), 501); + + svc.shutdown().await; +} + +#[tokio::test] +async fn tokenize_defaults_to_the_only_registered_model() { + let svc = Service::start(FIXTURE_COMPLETION, MODEL_COMPLETION).await; + + let body = svc.post_ok("/tokenize", json!({"prompt": "hi"})).await; + assert!(!tokens(&body).is_empty()); + + svc.shutdown().await; +} + +#[tokio::test] +async fn tokenize_unknown_model_is_404() { + let svc = Service::start(FIXTURE_COMPLETION, MODEL_COMPLETION).await; + + for path in ["/tokenize", "/detokenize"] { + let body = match path { + "/tokenize" => json!({"model": "does-not-exist", "prompt": "hi"}), + _ => json!({"model": "does-not-exist", "tokens": [1, 2, 3]}), + }; + assert_eq!(svc.post(path, body).await.status(), 404, "{path}"); + } + + svc.shutdown().await; +} + +#[tokio::test] +async fn detokenize_round_trips() { + let svc = Service::start(FIXTURE_COMPLETION, MODEL_COMPLETION).await; + + let encoded = svc + .post_ok( + "/tokenize", + json!({ + "model": MODEL_COMPLETION, + "prompt": "Hello, world!", + "add_special_tokens": false, + }), + ) + .await; + let decoded = svc + .post_ok( + "/detokenize", + json!({"model": MODEL_COMPLETION, "tokens": encoded["tokens"]}), + ) + .await; + + assert_eq!(decoded["prompt"].as_str(), Some("Hello, world!")); + + svc.shutdown().await; +} + +#[tokio::test] +async fn malformed_body_is_a_json_400() { + let svc = Service::start(FIXTURE_COMPLETION, MODEL_COMPLETION).await; + + // Matches neither arm of the untagged union. + let resp = svc + .post( + "/tokenize", + json!({"model": MODEL_COMPLETION, "text": "hi"}), + ) + .await; + assert_eq!(resp.status(), 400); + let body: Value = resp.json().await.expect("errors must stay JSON"); + let message = body["message"].as_str().expect("message field"); + assert!( + message.contains("`prompt`") && message.contains("`messages`"), + "the error should name the two accepted shapes: {message}" + ); + + assert_eq!( + svc.post("/detokenize", json!({"model": MODEL_COMPLETION})) + .await + .status(), + 400 + ); + + svc.shutdown().await; +} + +#[tokio::test] +async fn tokenize_chat_rejects_multimodal_content() { + let svc = Service::start(FIXTURE_CHAT, MODEL_CHAT).await; + + // The template's single placeholder is nowhere near what the image expands to. + let resp = svc + .post( + "/tokenize", + json!({ + "model": MODEL_CHAT, + "messages": [{"role": "user", "content": [ + {"type": "text", "text": "what is this?"}, + {"type": "image_url", "image_url": {"url": "https://example.com/a.png"}}, + ]}], + }), + ) + .await; + assert_eq!(resp.status(), 501); + + // A content array that is only text parts is ordinary text and must still work. + let body = svc + .post_ok( + "/tokenize", + json!({ + "model": MODEL_CHAT, + "messages": [{"role": "user", "content": [{"type": "text", "text": "hello"}]}], + }), + ) + .await; + assert!(!tokens(&body).is_empty()); + + svc.shutdown().await; +} + +#[tokio::test] +async fn token_strs_is_null_rather_than_absent() { + let svc = Service::start(FIXTURE_COMPLETION, MODEL_COMPLETION).await; + + // vLLM dumps the field unconditionally; clients read resp["token_strs"] directly. + let body = svc + .post_ok( + "/tokenize", + json!({"model": MODEL_COMPLETION, "prompt": "hi"}), + ) + .await; + assert_eq!(body.get("token_strs"), Some(&Value::Null)); + + // vLLM types the field `bool | None`, so an explicit null must parse. + let body = svc + .post_ok( + "/tokenize", + json!({"model": MODEL_COMPLETION, "prompt": "hi", "return_token_strs": null}), + ) + .await; + assert_eq!(body.get("token_strs"), Some(&Value::Null)); + + svc.shutdown().await; +} + +#[tokio::test] +async fn detokenize_keeps_special_tokens() { + let svc = Service::start(FIXTURE_CHAT, MODEL_CHAT).await; + + // vLLM does not skip special tokens on /detokenize, so a chat prompt's own specials — + // the ones its template inserted — have to survive the round-trip. + let encoded = svc + .post_ok( + "/tokenize", + json!({ + "model": MODEL_CHAT, + "messages": [{"role": "user", "content": "hi"}], + "return_token_strs": true, + }), + ) + .await; + let first = encoded["token_strs"][0] + .as_str() + .expect("leading token str") + .to_string(); + assert!( + first.starts_with('<'), + "expected a special token, got {first:?}" + ); + + let decoded = svc + .post_ok( + "/detokenize", + json!({"model": MODEL_CHAT, "tokens": encoded["tokens"]}), + ) + .await; + assert!( + decoded["prompt"] + .as_str() + .expect("prompt") + .starts_with(&first), + "the template's special token must survive detokenize" + ); + + svc.shutdown().await; +} + +/// A chat template whose output length depends on `enable_thinking`, so a token-count +/// difference is proof the flag reached the renderer. +const THINKING_TEMPLATE: &str = r#"{% for m in messages %}{{ m['content'] }}{% endfor %}{% if enable_thinking %} thinking thinking thinking{% endif %}"#; + +fn thinking_template_file() -> tempfile::TempPath { + use std::io::Write; + let mut f = tempfile::Builder::new() + .suffix(".jinja") + .tempfile() + .expect("tempfile"); + f.write_all(THINKING_TEMPLATE.as_bytes()).expect("write"); + f.into_temp_path() +} + +async fn thinking_service(default_mode: &str) -> Service { + let mode = default_mode.to_string(); + Service::start_with( + FIXTURE_COMPLETION, + MODEL_COMPLETION, + Some(thinking_template_file()), + move |card| { + card.runtime_config + .runtime_data + .insert("default_thinking_mode".to_string(), json!(mode)); + }, + ) + .await +} + +#[tokio::test] +async fn tokenize_chat_applies_the_models_default_thinking_mode() { + let body = json!({"model": MODEL_COMPLETION, "messages": [{"role": "user", "content": "hi"}]}); + + // The generate path injects the model's default thinking mode when the client sends no + // thinking control; /tokenize has to render the same prompt the model would be sent. + let enabled = thinking_service("enabled").await; + let on = enabled.post_ok("/tokenize", body.clone()).await; + enabled.shutdown().await; + + let disabled = thinking_service("disabled").await; + let off = disabled.post_ok("/tokenize", body.clone()).await; + + assert!( + tokens(&on).len() > tokens(&off).len(), + "default_thinking_mode must reach the template: on={} off={}", + tokens(&on).len(), + tokens(&off).len() + ); + + // An explicit client value still wins over the model default. + let mut explicit = body; + explicit["chat_template_kwargs"] = json!({"enable_thinking": true}); + let overridden = disabled.post_ok("/tokenize", explicit).await; + assert_eq!(tokens(&overridden).len(), tokens(&on).len()); + + disabled.shutdown().await; +} + +#[tokio::test] +async fn tokenize_requires_the_models_pipeline_preprocessor() { + // A card alone is not enough. Falling back to a tokenizer built here is exactly the + // drift this plumbing exists to prevent, so the absence has to be an error. + let (listener, port) = bind_random_port().await; + let service = HttpService::builder() + .port(port) + .host("127.0.0.1") + .build() + .expect("build"); + let mut card = ModelDeploymentCard::load_from_disk(FIXTURE_COMPLETION, None).expect("card"); + card.display_name = MODEL_COMPLETION.to_string(); + service + .model_manager() + .save_model_card("card-only", card) + .expect("save_model_card"); + let cancel = CancellationToken::new(); + let join = service.spawn_with_listener(cancel.clone(), listener).await; + + let client = reqwest::Client::builder() + .no_proxy() + .build() + .expect("client"); + for (path, body) in [ + ( + "/tokenize", + json!({"model": MODEL_COMPLETION, "prompt": "hi"}), + ), + ( + "/detokenize", + json!({"model": MODEL_COMPLETION, "tokens": [1]}), + ), + ] { + let resp = client + .post(format!("http://127.0.0.1:{port}{path}")) + .json(&body) + .send() + .await + .expect("send"); + assert_eq!(resp.status(), 501, "{path}"); + } + + cancel.cancel(); + let _ = join.await; +}