@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 69fdaffbcd

Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

[Rust Frontend] Add /tokenize and /detokenize endpoints (#44222)

Signed-off-by: Tan Ngoc Do <darkknightkhtn2008@gmail.com> Signed-off-by: TanNgocDo <darkknightkhtn2008@gmail.com> Signed-off-by: Bugen Zhao <i@bugenzhao.com> Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com> Co-authored-by: Bugen Zhao <i@bugenzhao.com>

TanNgocDo committed Jun 9, 2026 at 19:11 UTC 69fdaffbcd8cb406a9750685370a831fe8978a0a

9 files changed +836 -39

rust/src/chat/src/error.rs

@@ -66,6 +66,8 @@ pub enum Error {

66 ToolCallStreamInvariant { message: String },

67 #[error(transparent)]

68 Text(#[from] vllm_text::Error),

69 + #[error(transparent)]

70 + Tokenizer(#[from] vllm_tokenizer::TokenizerError),

71 }

73 pub type Result<T> = std::result::Result<T, Error>;

rust/src/chat/src/lib.rs

+28 -1

@@ -52,7 +52,7 @@ mod stream;

52 use vllm_engine_core_client::EngineCoreClient;

53 use vllm_engine_core_client::protocol::ModelDtype;

54 use vllm_llm::Llm;

55 -use vllm_text::{TextLlm, TextRequest};

55 +use vllm_text::{Prompt, TextLlm, TextRequest};

57 /// Validate explicit parser override names without starting request processing.

58 pub fn validate_parser_overrides(

@@ -198,6 +198,33 @@ impl ChatLlm {

198 Ok(ChatEventStream::new(request.request_id, structured_stream))

199 }

200

201 + /// Render through the chat template and tokenize, without submitting to the engine.

202 + ///

203 + /// Same render → [`multimodal::finalize_rendered_prompt`] → encode pipeline as

204 + /// [`Self::chat`], but stops after token IDs so `/tokenize` counts match what

205 + /// generation would see. Used by `POST /tokenize` (chat form).

206 + pub async fn tokenize_chat(&self, request: ChatRequest) -> Result<Vec<u32>> {

207 + request.validate()?;

208 +

209 + let rendered = self.backend.chat_renderer().render(&request)?;

210 + let (prompt, _mm_features) = multimodal::finalize_rendered_prompt(

211 + &request,

212 + rendered,

213 + self.backend.multimodal_model_info(),

214 + self.model_dtype,

215 + )

216 + .await?;

217 +

218 + let tokenizer = self.text.tokenizer();

219 + let token_ids = match prompt {

220 + // Rendered string from the template (usual chat path).

221 + Prompt::Text(text) => tokenizer.encode(&text, request.add_special_tokens)?,

222 + // Already tokenized (e.g. multimodal path); pass through unchanged.

223 + Prompt::TokenIds(ids) => ids,

224 + };

225 + Ok(token_ids)

226 + }

227 +

228 /// Shut down the underlying LLM client and its background tasks.

229 pub async fn shutdown(self) -> Result<()> {

230 self.text.shutdown().await?;

rust/src/server/src/routes.rs

+4 -1

@@ -9,6 +9,7 @@ pub(crate) mod openai;

9 mod pause;

10 mod server_info;

11 mod sleep;

12 +mod tokenize;

13 mod version;

15 use std::sync::Arc;

@@ -72,7 +73,9 @@ fn build_router_with_options(

73 .route("/v1/models", get(openai::list_models))

74 .route("/v1/completions", post(openai::completions))

75 .route("/v1/chat/completions", post(openai::chat_completions))

75 - // vLLM specific inference endpoints

76 + // vLLM specific endpoints

77 + .route("/tokenize", post(tokenize::tokenize))

78 + .route("/detokenize", post(tokenize::detokenize))

79 .route("/inference/v1/generate", post(inference::generate));

81 if runtime_lora_updating_enabled {

rust/src/server/src/routes/openai/chat_completions/convert.rs

+3 -4

@@ -156,8 +156,7 @@ pub(crate) fn prepare_chat_request(

156 return_tokens_as_token_ids: request.return_tokens_as_token_ids.unwrap_or(false),

157 })

158 }

159 -

160 -fn normalize_generation_prompt_mode(

159 +pub(crate) fn normalize_generation_prompt_mode(

160 add_generation_prompt: Option<bool>,

161 continue_final_message: bool,

162 messages: &[VllmChatMessage],

@@ -204,7 +203,7 @@ fn extract_last_assistant_content(messages: &[ChatMessage]) -> Option<String> {

203 }

204

205 /// Lower one OpenAI chat message into the `vllm-chat` message shape.

207 -fn convert_message(message: ChatMessage) -> Result<VllmChatMessage, ApiError> {

206 +pub(crate) fn convert_message(message: ChatMessage) -> Result<VllmChatMessage, ApiError> {

207 match message {

208 ChatMessage::System { content, .. } => {

209 Ok(VllmChatMessage::system(convert_content(content)?))

@@ -316,7 +315,7 @@ fn convert_assistant_tool_calls(

315 .collect()

316 }

317

319 -fn convert_tools(tools: Option<Vec<Tool>>) -> Result<Vec<ChatTool>, ApiError> {

318 +pub(crate) fn convert_tools(tools: Option<Vec<Tool>>) -> Result<Vec<ChatTool>, ApiError> {

319 tools

320 .unwrap_or_default()

321 .into_iter()

rust/src/server/src/routes/openai/chat_completions/types.rs

+3 -29

@@ -9,9 +9,9 @@ use vllm_chat::ReasoningEffort;

10 use crate::routes::openai::utils::structured_outputs::ResponseFormat;

11 use crate::routes::openai::utils::types::{

12 - ChatLogProbs, ChatMessage, MessageContent, Normalizable, StreamOptions, StringOrArray, Tool,

13 - ToolCall, ToolCallDelta, ToolChoice, ToolChoiceValue, ToolReference, UNKNOWN_MODEL_ID, Usage,

14 - default_true, validate_stop, validate_top_p_value,

12 + ChatLogProbs, ChatMessage, Normalizable, StreamOptions, StringOrArray, Tool, ToolCall,

13 + ToolCallDelta, ToolChoice, ToolChoiceValue, ToolReference, UNKNOWN_MODEL_ID, Usage,

14 + default_true, validate_messages, validate_stop, validate_top_p_value,

15 };

17 /// vLLM-compatible request type for the Chat Completions API.

@@ -430,32 +430,6 @@ fn default_model() -> String {

430 UNKNOWN_MODEL_ID.to_string()

431 }

432

433 -/// Validates messages array is not empty and has valid content

434 -fn validate_messages(messages: &[ChatMessage]) -> Result<(), validator::ValidationError> {

435 - if messages.is_empty() {

436 - return Err(validator::ValidationError::new("messages cannot be empty"));

437 - }

438 -

439 - for msg in messages {

440 - if let ChatMessage::User { content, .. } = msg {

441 - match content {

442 - MessageContent::Text(text) if text.is_empty() => {

443 - return Err(validator::ValidationError::new(

444 - "message content cannot be empty",

445 - ));

446 - }

447 - MessageContent::Parts(parts) if parts.is_empty() => {

448 - return Err(validator::ValidationError::new(

449 - "message content parts cannot be empty",

450 - ));

451 - }

452 - _ => {}

453 - }

454 - }

455 - }

456 - Ok(())

457 -}

458 -

433 /// Schema-level validation for cross-field dependencies

434 fn validate_chat_cross_parameters(

435 req: &ChatCompletionRequest,

rust/src/server/src/routes/openai/utils/types.rs

+35

@@ -412,6 +412,41 @@ pub struct ListModelsResponse {

412 pub data: Vec<ModelObject>,

413 }

414

415 +// ============================================================================

416 +// Shared validation helpers

417 +// ============================================================================

418 +

419 +/// Validates a messages array is non-empty and has valid user-message content.

420 +///

421 +/// Used by both `POST /v1/chat/completions` and `POST /tokenize` (chat form)

422 +/// so validation behaviour stays in lockstep.

423 +pub(crate) fn validate_messages(

424 + messages: &[ChatMessage],

425 +) -> Result<(), validator::ValidationError> {

426 + if messages.is_empty() {

427 + return Err(validator::ValidationError::new("messages cannot be empty"));

428 + }

429 +

430 + for msg in messages {

431 + if let ChatMessage::User { content, .. } = msg {

432 + match content {

433 + MessageContent::Text(text) if text.is_empty() => {

434 + return Err(validator::ValidationError::new(

435 + "message content cannot be empty",

436 + ));

437 + }

438 + MessageContent::Parts(parts) if parts.is_empty() => {

439 + return Err(validator::ValidationError::new(

440 + "message content parts cannot be empty",

441 + ));

442 + }

443 + _ => {}

444 + }

445 + }

446 + }

447 + Ok(())

448 +}

449 +

450 // ============================================================================

451 // Normalizable trait

452 // ============================================================================

rust/src/server/src/routes/tests.rs

+404 -4

@@ -407,6 +407,9 @@ struct FakeChatBackend {

407 multimodal_model_info: Option<vllm_chat::multimodal::MultimodalModelInfo>,

408 }

409

410 +/// Synthetic BOS id used when `add_special_tokens` is true in tests.

411 +const FAKE_BOS_TOKEN_ID: u32 = 1;

412 +

413 #[derive(Debug)]

414 struct FakeChatTokenizer;

415

@@ -414,9 +417,12 @@ impl Tokenizer for FakeChatTokenizer {

417 fn encode(

418 &self,

419 text: &str,

417 - _add_special_tokens: bool,

420 + add_special_tokens: bool,

421 ) -> vllm_text::tokenizer::Result<Vec<u32>> {

422 let mut token_ids = Vec::new();

423 + if add_special_tokens {

424 + token_ids.push(FAKE_BOS_TOKEN_ID);

425 + }

426 let mut rest = text;

427 while !rest.is_empty() {

428 if let Some(stripped) = rest.strip_prefix("<image>") {

@@ -460,6 +466,7 @@ impl Tokenizer for FakeChatTokenizer {

466

467 fn id_to_token(&self, id: u32) -> Option<String> {

468 match id {

469 + FAKE_BOS_TOKEN_ID => Some("<bos>".to_string()),

470 999 => Some("<image>".to_string()),

471 151655 => Some("<|image_pad|>".to_string()),

472 0xF001 => Some("<think>".to_string()),

@@ -468,6 +475,7 @@ impl Tokenizer for FakeChatTokenizer {

475 0xF004 => Some("<|END_THINKING|>".to_string()),

476 0xF005 => Some("◁think▷".to_string()),

477 0xF006 => Some("◁/think▷".to_string()),

478 + id if id < 128 => char::from_u32(id).map(|ch| ch.to_string()),

479 _ => None,

480 }

481 }

@@ -2442,7 +2450,8 @@ async fn non_stream_completions_echo_prepends_prompt_text() {

2450 "model": "Qwen/Qwen1.5-0.5B-Chat",

2451 "prompt": "hello",

2452 "echo": true,

2445 - "stream": false

2453 + "stream": false,

2454 + "add_special_tokens": false

2455 })

2456 .to_string(),

2457 ))

@@ -2645,7 +2654,8 @@ async fn non_stream_completions_include_prompt_logprobs() {

2654 "prompt": "hello",

2655 "stream": false,

2656 "echo": true,

2648 - "logprobs": 1

2657 + "logprobs": 1,

2658 + "add_special_tokens": false

2659 })

2660 .to_string(),

2661 ))

@@ -3426,7 +3436,8 @@ async fn completions_echo_stream_emits_separate_prompt_chunk() {

3436 "prompt": "hello",

3437 "echo": true,

3438 "stream": true,

3429 - "stream_options": {"include_usage": true}

3439 + "stream_options": {"include_usage": true},

3440 + "add_special_tokens": false

3441 })

3442 .to_string(),

3443 ))

@@ -4864,3 +4875,392 @@ async fn completions_empty_stop_string_returns_validation_error() {

4875

4876 assert_eq!(response.status(), StatusCode::BAD_REQUEST);

4877 }

4878 +

4879 +async fn post_json(

4880 + app: &mut axum::Router,

4881 + uri: &str,

4882 + body: serde_json::Value,

4883 +) -> (StatusCode, serde_json::Value) {

4884 + let response = app

4885 + .call(

4886 + Request::builder()

4887 + .method("POST")

4888 + .uri(uri)

4889 + .header("content-type", "application/json")

4890 + .body(Body::from(body.to_string()))

4891 + .expect("build request"),

4892 + )

4893 + .await

4894 + .expect("call app");

4895 + let status = response.status();

4896 + let bytes = to_bytes(response.into_body(), usize::MAX).await.expect("read body");

4897 + let json: serde_json::Value = serde_json::from_slice(&bytes)

4898 + .unwrap_or_else(|_| json!({ "raw": String::from_utf8_lossy(&bytes) }));

4899 + (status, json)

4900 +}

4901 +

4902 +#[tokio::test(flavor = "multi_thread", worker_threads = 2)]

4903 +#[serial]

4904 +async fn tokenize_completion_round_trips_through_detokenize() {

4905 + let mut app = test_app().await;

4906 + let prompt = "Hello world";

4907 +

4908 + let (_, tokenize_json) = post_json(

4909 + &mut app,

4910 + "/tokenize",

4911 + json!({

4912 + "model": "Qwen/Qwen1.5-0.5B-Chat",

4913 + "prompt": prompt,

4914 + "add_special_tokens": false,

4915 + }),

4916 + )

4917 + .await;

4918 + let tokens = tokenize_json["tokens"]

4919 + .as_array()

4920 + .expect("tokens array")

4921 + .iter()

4922 + .map(|v| v.as_u64().expect("token id") as u32)

4923 + .collect::<Vec<_>>();

4924 +

4925 + let (status, detokenize_json) = post_json(

4926 + &mut app,

4927 + "/detokenize",

4928 + json!({

4929 + "model": "Qwen/Qwen1.5-0.5B-Chat",

4930 + "tokens": tokens,

4931 + }),

4932 + )

4933 + .await;

4934 +

4935 + assert_eq!(status, StatusCode::OK);

4936 + assert_eq!(detokenize_json["prompt"], prompt);

4937 +}

4938 +

4939 +#[tokio::test(flavor = "multi_thread", worker_threads = 2)]

4940 +#[serial]

4941 +async fn tokenize_completion_add_special_tokens_changes_ids() {

4942 + let mut app = test_app().await;

4943 +

4944 + let (_, with_special) = post_json(

4945 + &mut app,

4946 + "/tokenize",

4947 + json!({

4948 + "model": "Qwen/Qwen1.5-0.5B-Chat",

4949 + "prompt": "hi",

4950 + "add_special_tokens": true,

4951 + }),

4952 + )

4953 + .await;

4954 + let (_, without_special) = post_json(

4955 + &mut app,

4956 + "/tokenize",

4957 + json!({

4958 + "model": "Qwen/Qwen1.5-0.5B-Chat",

4959 + "prompt": "hi",

4960 + "add_special_tokens": false,

4961 + }),

4962 + )

4963 + .await;

4964 +

4965 + let with_ids: Vec<u32> = with_special["tokens"]

4966 + .as_array()

4967 + .unwrap()

4968 + .iter()

4969 + .map(|v| v.as_u64().unwrap() as u32)

4970 + .collect();

4971 + let without_ids: Vec<u32> = without_special["tokens"]

4972 + .as_array()

4973 + .unwrap()

4974 + .iter()

4975 + .map(|v| v.as_u64().unwrap() as u32)

4976 + .collect();

4977 +

4978 + assert_ne!(with_ids, without_ids);

4979 + assert_eq!(with_ids.first().copied(), Some(FAKE_BOS_TOKEN_ID));

4980 + assert_eq!(without_ids.first().copied(), Some(b'h' as u32));

4981 +}

4982 +

4983 +#[tokio::test(flavor = "multi_thread", worker_threads = 2)]

4984 +#[serial]

4985 +async fn tokenize_completion_return_token_strs_matches_tokens() {

4986 + let mut app = test_app().await;

4987 +

4988 + let (status, json) = post_json(

4989 + &mut app,

4990 + "/tokenize",

4991 + json!({

4992 + "model": "Qwen/Qwen1.5-0.5B-Chat",

4993 + "prompt": "hi",

4994 + "add_special_tokens": false,

4995 + "return_token_strs": true,

4996 + }),

4997 + )

4998 + .await;

4999 +

5000 + assert_eq!(status, StatusCode::OK);

5001 + let tokens = json["tokens"].as_array().expect("tokens");

5002 + let token_strs = json["token_strs"].as_array().expect("token_strs");

5003 + assert_eq!(tokens.len(), token_strs.len());

5004 + assert_eq!(token_strs.len(), json["count"].as_u64().unwrap() as usize);

5005 + assert!(!token_strs[0].as_str().unwrap().is_empty());

5006 +}

5007 +

5008 +#[tokio::test(flavor = "multi_thread", worker_threads = 2)]

5009 +#[serial]

5010 +async fn tokenize_completion_count_and_max_model_len() {

5011 + let mut app = test_app().await;

5012 +

5013 + let (status, json) = post_json(

5014 + &mut app,

5015 + "/tokenize",

5016 + json!({

5017 + "model": "Qwen/Qwen1.5-0.5B-Chat",

5018 + "prompt": "hello",

5019 + "add_special_tokens": false,

5020 + }),

5021 + )

5022 + .await;

5023 +

5024 + assert_eq!(status, StatusCode::OK);

5025 + assert_eq!(

5026 + json["count"].as_u64().unwrap() as usize,

5027 + json["tokens"].as_array().unwrap().len()

5028 + );

5029 + assert!(json["max_model_len"].as_u64().unwrap() > 0);

5030 +}

5031 +

5032 +#[tokio::test(flavor = "multi_thread", worker_threads = 2)]

5033 +#[serial]

5034 +async fn tokenize_chat_includes_generation_prompt_in_token_count() {

5035 + let mut app = test_app().await;

5036 + let messages = json!([{"role": "user", "content": "hi"}]);

5037 +

5038 + let (_, with_prompt) = post_json(

5039 + &mut app,

5040 + "/tokenize",

5041 + json!({

5042 + "model": "Qwen/Qwen1.5-0.5B-Chat",

5043 + "messages": messages,

5044 + "add_generation_prompt": true,

5045 + "add_special_tokens": false,

5046 + }),

5047 + )

5048 + .await;

5049 + let (_, without_prompt) = post_json(

5050 + &mut app,

5051 + "/tokenize",

5052 + json!({

5053 + "model": "Qwen/Qwen1.5-0.5B-Chat",

5054 + "messages": messages,

5055 + "add_generation_prompt": false,

5056 + "add_special_tokens": false,

5057 + }),

5058 + )

5059 + .await;

5060 +

5061 + let with_len = with_prompt["tokens"].as_array().unwrap().len();

5062 + let without_len = without_prompt["tokens"].as_array().unwrap().len();

5063 + assert!(with_len > without_len);

5064 +}

5065 +

5066 +#[tokio::test(flavor = "multi_thread", worker_threads = 2)]

5067 +#[serial]

5068 +async fn tokenize_chat_conflicting_generation_flags_returns_400() {

5069 + let mut app = test_app().await;

5070 +

5071 + let (status, json) = post_json(

5072 + &mut app,

5073 + "/tokenize",

5074 + json!({

5075 + "model": "Qwen/Qwen1.5-0.5B-Chat",

5076 + "messages": [{"role": "user", "content": "hi"}],

5077 + "add_generation_prompt": true,

5078 + "continue_final_message": true,

5079 + }),

5080 + )

5081 + .await;

5082 +

5083 + assert_eq!(status, StatusCode::BAD_REQUEST);

5084 + assert_eq!(json["error"]["type"], "invalid_request_error");

5085 +}

5086 +

5087 +#[tokio::test(flavor = "multi_thread", worker_threads = 2)]

5088 +#[serial]

5089 +async fn tokenize_chat_empty_messages_returns_400() {

5090 + let mut app = test_app().await;

5091 +

5092 + let (status, json) = post_json(

5093 + &mut app,

5094 + "/tokenize",

5095 + json!({

5096 + "model": "Qwen/Qwen1.5-0.5B-Chat",

5097 + "messages": [],

5098 + }),

5099 + )

5100 + .await;

5101 +

5102 + assert_eq!(status, StatusCode::BAD_REQUEST);

5103 + assert_eq!(json["error"]["type"], "invalid_request_error");

5104 +}

5105 +

5106 +#[tokio::test(flavor = "multi_thread", worker_threads = 2)]

5107 +#[serial]

5108 +async fn tokenize_chat_empty_message_content_returns_400() {

5109 + let mut app = test_app().await;

5110 +

5111 + let (status, json) = post_json(

5112 + &mut app,

5113 + "/tokenize",

5114 + json!({

5115 + "model": "Qwen/Qwen1.5-0.5B-Chat",

5116 + "messages": [{"role": "user", "content": ""}],

5117 + }),

5118 + )

5119 + .await;

5120 +

5121 + assert_eq!(status, StatusCode::BAD_REQUEST);

5122 + assert_eq!(json["error"]["type"], "invalid_request_error");

5123 +}

5124 +

5125 +#[tokio::test(flavor = "multi_thread", worker_threads = 2)]

5126 +#[serial]

5127 +async fn tokenize_unknown_model_returns_404() {

5128 + let mut app = test_app().await;

5129 +

5130 + let (status, json) = post_json(

5131 + &mut app,

5132 + "/tokenize",

5133 + json!({

5134 + "model": "does-not-exist",

5135 + "prompt": "hello",

5136 + }),

5137 + )

5138 + .await;

5139 +

5140 + assert_eq!(status, StatusCode::NOT_FOUND);

5141 + assert_eq!(json["error"]["type"], "invalid_request_error");

5142 + assert_eq!(json["error"]["code"], "model_not_found");

5143 +}

5144 +

5145 +#[tokio::test(flavor = "multi_thread", worker_threads = 2)]

5146 +#[serial]

5147 +async fn detokenize_unknown_model_returns_404() {

5148 + let mut app = test_app().await;

5149 +

5150 + let (status, json) = post_json(

5151 + &mut app,

5152 + "/detokenize",

5153 + json!({

5154 + "model": "does-not-exist",

5155 + "tokens": [72, 101, 108, 108, 111],

5156 + }),

5157 + )

5158 + .await;

5159 +

5160 + assert_eq!(status, StatusCode::NOT_FOUND);

5161 + assert_eq!(json["error"]["type"], "invalid_request_error");

5162 + assert_eq!(json["error"]["code"], "model_not_found");

5163 +}

5164 +

5165 +#[tokio::test(flavor = "multi_thread", worker_threads = 2)]

5166 +#[serial]

5167 +async fn detokenize_empty_tokens_returns_empty_prompt() {

5168 + let mut app = test_app().await;

5169 +

5170 + let (status, json) = post_json(

5171 + &mut app,

5172 + "/detokenize",

5173 + json!({

5174 + "model": "Qwen/Qwen1.5-0.5B-Chat",

5175 + "tokens": [],

5176 + }),

5177 + )

5178 + .await;

5179 +

5180 + assert_eq!(status, StatusCode::OK);

5181 + assert_eq!(json["prompt"], "");

5182 +}

5183 +

5184 +/// Decode an explicit token sequence — pins `/detokenize` independently of

5185 +/// `/tokenize` (the round-trip test alone would pass even if encode and decode

5186 +/// were both wrong in mirrored ways).

5187 +#[tokio::test(flavor = "multi_thread", worker_threads = 2)]

5188 +#[serial]

5189 +async fn detokenize_decodes_known_token_ids() {

5190 + let mut app = test_app().await;

5191 +

5192 + let (status, json) = post_json(

5193 + &mut app,

5194 + "/detokenize",

5195 + json!({

5196 + "model": "Qwen/Qwen1.5-0.5B-Chat",

5197 + "tokens": [72, 101, 108, 108, 111],

5198 + }),

5199 + )

5200 + .await;

5201 +

5202 + assert_eq!(status, StatusCode::OK);

5203 + assert_eq!(json["prompt"], "Hello");

5204 +}

5205 +

5206 +/// `continue_final_message` without a trailing assistant message must 400.

5207 +#[tokio::test(flavor = "multi_thread", worker_threads = 2)]

5208 +#[serial]

5209 +async fn tokenize_chat_continue_without_assistant_returns_400() {

5210 + let mut app = test_app().await;

5211 +

5212 + let (status, json) = post_json(

5213 + &mut app,

5214 + "/tokenize",

5215 + json!({

5216 + "model": "Qwen/Qwen1.5-0.5B-Chat",

5217 + "messages": [{"role": "user", "content": "hi"}],

5218 + "add_generation_prompt": false,

5219 + "continue_final_message": true,

5220 + }),

5221 + )

5222 + .await;

5223 +

5224 + assert_eq!(status, StatusCode::BAD_REQUEST);

5225 + assert_eq!(json["error"]["type"], "invalid_request_error");

5226 +}

5227 +

5228 +/// `continue_final_message` must not append a new generation suffix vs `add_generation_prompt`.

5229 +#[tokio::test(flavor = "multi_thread", worker_threads = 2)]

5230 +#[serial]

5231 +async fn tokenize_chat_continue_final_vs_new_assistant_differs() {

5232 + let mut app = test_app().await;

5233 + let messages = json!([

5234 + {"role": "user", "content": "hi"},

5235 + {"role": "assistant", "content": "partial,"}

5236 + ]);

5237 +

5238 + let (_, continue_final) = post_json(

5239 + &mut app,

5240 + "/tokenize",

5241 + json!({

5242 + "model": "Qwen/Qwen1.5-0.5B-Chat",

5243 + "messages": messages,

5244 + "add_generation_prompt": false,

5245 + "continue_final_message": true,

5246 + "add_special_tokens": false,

5247 + }),

5248 + )

5249 + .await;

5250 + let (_, new_assistant) = post_json(

5251 + &mut app,

5252 + "/tokenize",

5253 + json!({

5254 + "model": "Qwen/Qwen1.5-0.5B-Chat",

5255 + "messages": messages,

5256 + "add_generation_prompt": true,

5257 + "continue_final_message": false,

5258 + "add_special_tokens": false,

5259 + }),

5260 + )

5261 + .await;

5262 +

5263 + let continue_len = continue_final["tokens"].as_array().unwrap().len();

5264 + let new_len = new_assistant["tokens"].as_array().unwrap().len();

5265 + assert!(new_len > continue_len);

5266 +}

rust/src/server/src/routes/tokenize.rs new

+148

@@ -0,0 +1,148 @@

1 +//! `POST /tokenize` and `POST /detokenize` (root paths, matching Python).

2 +//!

3 +//! Encode/decode runs entirely in-process via [`DynTokenizer`]; the inference

4 +//! engine is not involved.

5 +

6 +mod types;

7 +

8 +use std::sync::Arc;

9 +

10 +use axum::Json;

11 +use axum::extract::State;

12 +use axum::http::HeaderMap;

13 +use axum::response::{IntoResponse, Response};

14 +use thiserror_ext::AsReport as _;

15 +

16 +use crate::error::{ApiError, server_error};

17 +use crate::routes::openai::utils::validated_json::ValidatedJson;

18 +use crate::routes::tokenize::types::{

19 + DetokenizeRequest, DetokenizeResponse, TokenizeChatRequest, TokenizeCompletionRequest,

20 + TokenizeRequest, TokenizeResponse,

21 +};

22 +use crate::state::AppState;

23 +use crate::utils::resolve_base_request_id;

24 +

25 +/// Match Python `tokenize-{base}` where base is `X-Request-Id` or a new UUID.

26 +fn tokenize_request_id(headers: &HeaderMap) -> String {

27 + let base = resolve_base_request_id(

28 + headers.get("X-Request-Id").and_then(|value| value.to_str().ok()),

29 + None,

30 + );

31 + format!("tokenize-{base}")

32 +}

33 +

34 +/// Reject an unknown model name, matching the other handlers.

35 +fn check_model(state: &AppState, model: Option<&str>) -> Result<(), ApiError> {

36 + if let Some(model) = model

37 + && !state.served_model_names().iter().any(|n| n == model)

38 + {

39 + return Err(ApiError::model_not_found(model.to_string()));

40 + }

41 + Ok(())

42 +}

43 +

44 +/// Build the `token_strs` vector when requested, via the tokenizer vocab.

45 +fn token_strs(tokenizer: &vllm_text::tokenizer::DynTokenizer, ids: &[u32]) -> Vec<String> {

46 + // Unknown IDs yield "" — intentional; matches Python's convert_ids_to_tokens behaviour.

47 + ids.iter().map(|&id| tokenizer.id_to_token(id).unwrap_or_default()).collect()

48 +}

49 +

50 +pub async fn tokenize(

51 + State(state): State<Arc<AppState>>,

52 + headers: HeaderMap,

53 + ValidatedJson(body): ValidatedJson<TokenizeRequest>,

54 +) -> Response {

55 + let request_id = tokenize_request_id(&headers);

56 + let tokenizer = state.chat.text().tokenizer();

57 + let max_model_len = state.chat.engine_core_client().max_model_len();

58 +

59 + let result = match body {

60 + // Completion form: encode the raw `prompt` string (no chat template).

61 + TokenizeRequest::Completion(req) => tokenize_completion(&state, &tokenizer, req),

62 + // Chat form: render `messages` through the template, then encode (see `tokenize_chat`).

63 + TokenizeRequest::Chat(req) => tokenize_chat(&state, &request_id, req).await,

64 + };

65 +

66 + match result {

67 + Ok((tokens, want_strs)) => {

68 + let token_strs = want_strs.then(|| token_strs(&tokenizer, &tokens));

69 + Json(TokenizeResponse {

70 + count: tokens.len(),

71 + max_model_len,

72 + tokens,

73 + token_strs,

74 + })

75 + .into_response()

76 + }

77 + Err(error) => error.into_response(),

78 + }

79 +}

80 +

81 +fn tokenize_completion(

82 + state: &AppState,

83 + tokenizer: &vllm_text::tokenizer::DynTokenizer,

84 + req: TokenizeCompletionRequest,

85 +) -> Result<(Vec<u32>, bool), ApiError> {

86 + check_model(state, req.model.as_deref())?;

87 + let tokens = tokenizer

88 + .encode(&req.prompt, req.add_special_tokens)

89 + .map_err(|e| server_error!("tokenize failed: {}", e.to_report_string()))?;

90 + Ok((tokens, req.return_token_strs))

91 +}

92 +

93 +/// HTTP adapter for the chat-shaped `/tokenize` body.

94 +///

95 +/// Not [`vllm_chat::ChatLlm::tokenize_chat`]: this checks the model name and maps

96 +/// errors to [`ApiError`]; the chat-crate method does render → finalize → encode.

97 +async fn tokenize_chat(

98 + state: &AppState,

99 + request_id: &str,

100 + req: TokenizeChatRequest,

101 +) -> Result<(Vec<u32>, bool), ApiError> {

102 + check_model(state, req.model.as_deref())?;

103 + let return_token_strs = req.return_token_strs;

104 + // `continue_final_message` / `add_generation_prompt` mutual exclusion is

105 + // enforced in `normalize_generation_prompt_mode` inside `into_chat_request`.

106 + let tokens = state

107 + .chat

108 + .tokenize_chat(req.into_chat_request(request_id.to_string())?)

109 + .await

110 + .map_err(|e| server_error!("tokenize failed: {}", e.to_report_string()))?;

111 + Ok((tokens, return_token_strs))

112 +}

113 +

114 +pub async fn detokenize(

115 + State(state): State<Arc<AppState>>,

116 + ValidatedJson(body): ValidatedJson<DetokenizeRequest>,

117 +) -> Response {

118 + if let Err(error) = check_model(&state, body.model.as_deref()) {

119 + return error.into_response();

120 + }

121 + let tokenizer = state.chat.text().tokenizer();

122 + match tokenizer.decode(&body.tokens, /* skip_special_tokens = */ false) {

123 + Ok(prompt) => Json(DetokenizeResponse { prompt }).into_response(),

124 + Err(e) => server_error!("detokenize failed: {}", e.to_report_string()).into_response(),

125 + }

126 +}

127 +

128 +#[cfg(test)]

129 +mod tests {

130 + use axum::http::{HeaderMap, HeaderValue};

131 +

132 + use super::tokenize_request_id;

133 +

134 + #[test]

135 + fn tokenize_request_id_prefers_x_request_id_header() {

136 + let mut headers = HeaderMap::new();

137 + headers.insert("X-Request-Id", HeaderValue::from_static("client-req-1"));

138 + assert_eq!(tokenize_request_id(&headers), "tokenize-client-req-1");

139 + }

140 +

141 + #[test]

142 + fn tokenize_request_id_generates_uuid_when_header_missing() {

143 + let headers = HeaderMap::new();

144 + let id = tokenize_request_id(&headers);

145 + assert!(id.starts_with("tokenize-"));

146 + assert_ne!(id, "tokenize-");

147 + }

148 +}

rust/src/server/src/routes/tokenize/types.rs new

+209

@@ -0,0 +1,209 @@

1 +use std::collections::HashMap;

2 +

3 +use itertools::Itertools as _;

4 +use serde::{Deserialize, Serialize};

5 +use serde_json::Value;

6 +use validator::{Validate, ValidationErrors};

7 +use vllm_chat::{ChatOptions, ChatRequest, ChatToolChoice, SamplingParams};

8 +use vllm_text::output::TextDecodeOptions;

9 +

10 +use crate::error::ApiError;

11 +use crate::routes::openai::chat_completions::convert::{

12 + convert_message, convert_tools, normalize_generation_prompt_mode,

13 +};

14 +use crate::routes::openai::utils::types::{

15 + ChatMessage, Normalizable, Tool, default_true, validate_messages,

16 +};

17 +

18 +/// `POST /tokenize` body. Untagged: a JSON object with `messages` parses as the

19 +/// chat variant; one with `prompt` parses as the completion variant.

20 +#[derive(Debug, Clone, Deserialize)]

21 +#[serde(untagged)]

22 +pub enum TokenizeRequest {

23 + Chat(TokenizeChatRequest),

24 + Completion(TokenizeCompletionRequest),

25 +}

26 +

27 +#[derive(Debug, Clone, Deserialize)]

28 +pub struct TokenizeCompletionRequest {

29 + pub model: Option<String>,

30 + pub prompt: String,

31 + #[serde(default = "default_true")]

32 + pub add_special_tokens: bool,

33 + #[serde(default)]

34 + pub return_token_strs: bool,

35 +}

36 +

37 +#[derive(Debug, Clone, Deserialize, Validate)]

38 +pub struct TokenizeChatRequest {

39 + pub model: Option<String>,

40 + #[validate(custom(function = "validate_messages"))]

41 + pub messages: Vec<ChatMessage>,

42 + #[serde(default = "default_true")]

43 + pub add_generation_prompt: bool,

44 + #[serde(default)]

45 + pub continue_final_message: bool,

46 + #[serde(default)] // chat default is FALSE (template adds specials)

47 + pub add_special_tokens: bool,

48 + #[serde(default)]

49 + pub return_token_strs: bool,

50 + #[serde(default)]

51 + pub chat_template: Option<String>,

52 + #[serde(default)]

53 + pub chat_template_kwargs: Option<HashMap<String, Value>>,

54 + #[serde(default)]

55 + pub tools: Option<Vec<Tool>>,

56 +}

57 +

58 +impl TokenizeChatRequest {

59 + /// Lower this tokenize body into a [`ChatRequest`] for template rendering.

60 + ///

61 + /// Reuses [`convert_message`] and [`normalize_generation_prompt_mode`] from

62 + /// `chat_completions/convert` so message lowering and generation-prompt

63 + /// rules match chat completions. Only fields that affect rendering are set;

64 + /// `sampling_params`, `decode_options`, etc. stay at default because

65 + /// tokenize never generates.

66 + pub fn into_chat_request(self, request_id: String) -> Result<ChatRequest, ApiError> {

67 + let messages: Vec<_> = self.messages.into_iter().map(convert_message).try_collect()?;

68 + let generation_prompt_mode = normalize_generation_prompt_mode(

69 + Some(self.add_generation_prompt),

70 + self.continue_final_message,

71 + &messages,

72 + )?;

73 +

74 + Ok(ChatRequest {

75 + request_id,

76 + messages,

77 + sampling_params: SamplingParams::default(),

78 + chat_options: ChatOptions {

79 + generation_prompt_mode,

80 + chat_template: self.chat_template,

81 + reasoning_effort: None,

82 + template_kwargs: self.chat_template_kwargs.unwrap_or_default(),

83 + },

84 + tools: convert_tools(self.tools)?,

85 + tool_choice: ChatToolChoice::Auto,

86 + decode_options: TextDecodeOptions::default(),

87 + intermediate: false,

88 + priority: 0,

89 + documents: None,

90 + cache_salt: None,

91 + add_special_tokens: self.add_special_tokens,

92 + data_parallel_rank: None,

93 + lora_request: None,

94 + })

95 + }

96 +}

97 +

98 +#[derive(Debug, Clone, Deserialize)]

99 +pub struct DetokenizeRequest {

100 + pub model: Option<String>,

101 + pub tokens: Vec<u32>,

102 +}

103 +

104 +#[derive(Debug, Clone, Serialize)]

105 +pub struct TokenizeResponse {

106 + pub count: usize,

107 + pub max_model_len: u32,

108 + pub tokens: Vec<u32>,

109 + #[serde(skip_serializing_if = "Option::is_none")]

110 + pub token_strs: Option<Vec<String>>,

111 +}

112 +

113 +#[derive(Debug, Clone, Serialize)]

114 +pub struct DetokenizeResponse {

115 + pub prompt: String,

116 +}

117 +

118 +// ---- trait impls required by ValidatedJson ----

119 +impl Validate for TokenizeRequest {

120 + fn validate(&self) -> Result<(), ValidationErrors> {

121 + if let Self::Chat(req) = self {

122 + req.validate()?;

123 + }

124 + Ok(())

125 + }

126 +}

127 +impl Validate for DetokenizeRequest {

128 + fn validate(&self) -> Result<(), ValidationErrors> {

129 + Ok(())

130 + }

131 +}

132 +impl Normalizable for TokenizeRequest {} // default no-op normalize()

133 +impl Normalizable for DetokenizeRequest {}

134 +

135 +#[cfg(test)]

136 +mod tests {

137 + use super::*;

138 + use crate::routes::openai::utils::types::{ChatMessage, MessageContent};

139 + use serde_json::json;

140 + use vllm_chat::ChatTool;

141 +

142 + #[test]

143 + fn tokenize_request_converts_openai_tools() {

144 + // The untagged `TokenizeRequest` must resolve a messages+tools body to

145 + // the chat variant and accept standard OpenAI tool objects

146 + // (`{"type":"function",...}`), then convert them to `ChatTool`.

147 + let request: TokenizeRequest = serde_json::from_value(json!({

148 + "messages": [{"role": "user", "content": "hi"}],

149 + "tools": [{

150 + "type": "function",

151 + "function": {

152 + "name": "get_weather",

153 + "description": "Get weather",

154 + "parameters": {

155 + "type": "object",

156 + "properties": {"city": {"type": "string"}},

157 + },

158 + },

159 + }],

160 + }))

161 + .expect("OpenAI tool JSON deserializes to the chat variant");

162 +

163 + let TokenizeRequest::Chat(req) = request else {

164 + panic!("messages+tools body should parse as the chat variant");

165 + };

166 +

167 + let chat_request =

168 + req.into_chat_request("tokenize-test".to_string()).expect("request is valid");

169 +

170 + assert_eq!(

171 + chat_request.tools,

172 + vec![ChatTool {

173 + name: "get_weather".to_string(),

174 + description: Some("Get weather".to_string()),

175 + parameters: json!({

176 + "type": "object",

177 + "properties": {"city": {"type": "string"}},

178 + }),

179 + strict: None,

180 + }]

181 + );

182 + }

183 +

184 + #[test]

185 + fn into_chat_request_rejects_conflicting_generation_flags() {

186 + let req = TokenizeChatRequest {

187 + model: None,

188 + messages: vec![ChatMessage::User {

189 + content: MessageContent::Text("hi".to_string()),

190 + name: None,

191 + }],

192 + add_generation_prompt: true,

193 + continue_final_message: true,

194 + add_special_tokens: false,

195 + return_token_strs: false,

196 + chat_template: None,

197 + chat_template_kwargs: None,

198 + tools: None,

199 + };

200 +

201 + let error = req

202 + .into_chat_request("tokenize-test".to_string())

203 + .expect_err("conflicting flags");

204 + assert_eq!(

205 + error.to_error_response().error.message,

206 + "Cannot set both `continue_final_message` and `add_generation_prompt` to True."

207 + );

208 + }

209 +}