@tduncan / Nvidia-Optimization-Stack-VLLM / commits / 7a327f0b4f

[Rust Frontend] Simplify unit tests with shared `TestTokenizer` (#47125)

Signed-off-by: Bugen Zhao <i@bugenzhao.com>

Bugen Zhao committed Jun 30, 2026 at 22:34 UTC 7a327f0b4f8f6886df77996d0e35ce547c9c2bd1

28 files changed +682 -828

rust/Cargo.lock

@@ -5310,6 +5310,7 @@ dependencies = [

5310 "vllm-llm",

5311 "vllm-metrics",

5312 "vllm-text",

5313 + "vllm-tokenizer",

5314 "zeromq",

5315 ]

5316

rust/src/chat/Cargo.toml

@@ -50,6 +50,7 @@ tokio.workspace = true

50 tracing-subscriber.workspace = true

51 uuid.workspace = true

52 vllm-engine-core-client = { workspace = true, features = ["test-util"] }

53 +vllm-tokenizer = { workspace = true, features = ["test-utils"] }

54 zeromq.workspace = true

56 [lints]

rust/src/chat/src/backend/hf.rs

+3 -26

@@ -154,7 +154,8 @@ mod tests {

154 use thiserror_ext::AsReport as _;

155 use vllm_text::Prompt;

156 use vllm_text::backend::hf::TokenizerSource;

157 - use vllm_text::tokenizer::{DynTokenizer, Tokenizer};

157 + use vllm_text::tokenizer::DynTokenizer;

158 + use vllm_tokenizer::test_utils::TestTokenizer;

159

160 use super::HfChatBackend;

161 use crate::backend::{ChatBackend, LoadModelBackendsOptions, NewChatOutputProcessorOptions};

@@ -196,32 +197,8 @@ mod tests {

197 }

198 }

199

199 - struct TestTokenizer;

200 -

201 - impl Tokenizer for TestTokenizer {

202 - fn encode(

203 - &self,

204 - _text: &str,

205 - _add_special_tokens: bool,

206 - ) -> vllm_text::tokenizer::Result<Vec<u32>> {

207 - Ok(Vec::new())

208 - }

209 -

210 - fn decode(

211 - &self,

212 - _token_ids: &[u32],

213 - _skip_special_tokens: bool,

214 - ) -> vllm_text::tokenizer::Result<String> {

215 - Ok(String::new())

216 - }

217 -

218 - fn token_to_id(&self, _token: &str) -> Option<u32> {

219 - None

220 - }

221 - }

222 -

200 fn test_tokenizer() -> DynTokenizer {

224 - Arc::new(TestTokenizer)

201 + Arc::new(TestTokenizer::new())

202 }

203

204 fn backend_for_selection(

rust/src/chat/src/multimodal.rs

+10 -56

@@ -563,7 +563,7 @@ mod tests {

563

564 use llm_multimodal::TokenId;

565 use vllm_engine_core_client::protocol::tensor::WireArrayData;

566 - use vllm_text::tokenizer::{IncrementalDecoder, Tokenizer, TokenizerError};

566 + use vllm_tokenizer::test_utils::TestTokenizer;

567

568 use super::*;

569

@@ -574,60 +574,14 @@ mod tests {

574 const LLAMA4_TILE_X_SEPARATOR_ID: u32 = 200093;

575 const LLAMA4_TILE_Y_SEPARATOR_ID: u32 = 200094;

576

577 - struct TestTokenizer;

578 -

579 - impl Tokenizer for TestTokenizer {

580 - fn encode(

581 - &self,

582 - text: &str,

583 - _add_special_tokens: bool,

584 - ) -> std::result::Result<Vec<u32>, TokenizerError> {

585 - Ok(match text {

586 - "<|image|>" => vec![LLAMA4_IMAGE_ID],

587 - text => text.bytes().map(u32::from).collect(),

588 - })

589 - }

590 -

591 - fn decode(

592 - &self,

593 - _token_ids: &[u32],

594 - _skip_special_tokens: bool,

595 - ) -> std::result::Result<String, TokenizerError> {

596 - Ok(String::new())

597 - }

598 -

599 - fn token_to_id(&self, token: &str) -> Option<u32> {

600 - match token {

601 - "<|image_start|>" => Some(LLAMA4_IMAGE_START_ID),

602 - "<|image_end|>" => Some(LLAMA4_IMAGE_END_ID),

603 - "<|image|>" => Some(LLAMA4_IMAGE_ID),

604 - "<|patch|>" => Some(LLAMA4_PATCH_ID),

605 - "<|tile_x_separator|>" => Some(LLAMA4_TILE_X_SEPARATOR_ID),

606 - "<|tile_y_separator|>" => Some(LLAMA4_TILE_Y_SEPARATOR_ID),

607 - _ => None,

608 - }

609 - }

610 -

611 - fn id_to_token(&self, id: u32) -> Option<String> {

612 - match id {

613 - LLAMA4_IMAGE_START_ID => Some("<|image_start|>".to_string()),

614 - LLAMA4_IMAGE_END_ID => Some("<|image_end|>".to_string()),

615 - LLAMA4_IMAGE_ID => Some("<|image|>".to_string()),

616 - LLAMA4_PATCH_ID => Some("<|patch|>".to_string()),

617 - LLAMA4_TILE_X_SEPARATOR_ID => Some("<|tile_x_separator|>".to_string()),

618 - LLAMA4_TILE_Y_SEPARATOR_ID => Some("<|tile_y_separator|>".to_string()),

619 - _ => None,

620 - }

621 - }

622 -

623 - fn create_decode_stream(

624 - &self,

625 - _prompt_token_ids: &[u32],

626 - _skip_special_tokens: bool,

627 - _min_bytes_to_buffer: usize,

628 - ) -> Box<dyn IncrementalDecoder + '_> {

629 - unreachable!("not used")

630 - }

577 + fn llama4_tokenizer() -> TestTokenizer {

578 + TestTokenizer::new()

579 + .with_regular_token("<|image_start|>", LLAMA4_IMAGE_START_ID)

580 + .with_regular_token("<|image_end|>", LLAMA4_IMAGE_END_ID)

581 + .with_regular_token("<|image|>", LLAMA4_IMAGE_ID)

582 + .with_regular_token("<|patch|>", LLAMA4_PATCH_ID)

583 + .with_regular_token("<|tile_x_separator|>", LLAMA4_TILE_X_SEPARATOR_ID)

584 + .with_regular_token("<|tile_y_separator|>", LLAMA4_TILE_Y_SEPARATOR_ID)

585 }

586

587 fn test_info(model_type: &str, config: serde_json::Value) -> MultimodalModelInfo {

@@ -635,7 +589,7 @@ mod tests {

589 model_id: format!("{model_type}-test"),

590 model_type: Some(model_type.to_string()),

591 config,

638 - tokenizer: TokenizerResolver(Arc::new(TestTokenizer)),

592 + tokenizer: TokenizerResolver(Arc::new(llama4_tokenizer())),

593 };

594 let spec = context

595 .resolve_model_spec()

rust/src/chat/src/output/default/mod.rs

+7 -34

@@ -189,46 +189,19 @@ impl ChatOutputProcessor for DefaultChatOutputProcessor {

189 mod tests {

190 use std::sync::Arc;

191

192 - use vllm_tokenizer::Tokenizer;

192 + use vllm_tokenizer::test_utils::TestTokenizer;

193

194 use super::DefaultChatOutputProcessor;

195 use crate::Error;

196 use crate::parser::ParserSelection;

197 use crate::request::ChatRequest;

198

199 - struct FakeTokenizer;

200 -

201 - impl Tokenizer for FakeTokenizer {

202 - fn encode(

203 - &self,

204 - text: &str,

205 - _add_special_tokens: bool,

206 - ) -> vllm_tokenizer::Result<Vec<u32>> {

207 - Ok(text.chars().map(u32::from).collect())

208 - }

209 -

210 - fn decode(

211 - &self,

212 - token_ids: &[u32],

213 - _skip_special_tokens: bool,

214 - ) -> vllm_tokenizer::Result<String> {

215 - Ok(token_ids

216 - .iter()

217 - .map(|token_id| char::from_u32(*token_id).unwrap_or('\u{FFFD}'))

218 - .collect())

219 - }

220 -

221 - fn token_to_id(&self, token: &str) -> Option<u32> {

222 - match token {

223 - "<|channel>" => Some(1),

224 - "<channel|>" => Some(2),

225 - _ => None,

226 - }

227 - }

228 - }

229 -

230 - fn tokenizer() -> Arc<FakeTokenizer> {

231 - Arc::new(FakeTokenizer)

199 + fn tokenizer() -> Arc<TestTokenizer> {

200 + Arc::new(

201 + TestTokenizer::new()

202 + .with_regular_token("<|channel>", 256)

203 + .with_regular_token("<channel|>", 257),

204 + )

205 }

206

207 #[test]

rust/src/chat/src/parser/reasoning/tests.rs

+2 -25

@@ -1,32 +1,9 @@

1 use std::sync::Arc;

3 -use vllm_tokenizer::Tokenizer;

3 +use vllm_tokenizer::test_utils::TestTokenizer;

5 use super::{ReasoningParserFactory, names};

7 -struct FakeTokenizer;

8 -

9 -impl Tokenizer for FakeTokenizer {

10 - fn encode(&self, text: &str, _add_special_tokens: bool) -> vllm_tokenizer::Result<Vec<u32>> {

11 - Ok(text.chars().map(u32::from).collect())

12 - }

13 -

14 - fn decode(

15 - &self,

16 - token_ids: &[u32],

17 - _skip_special_tokens: bool,

18 - ) -> vllm_tokenizer::Result<String> {

19 - Ok(token_ids

20 - .iter()

21 - .map(|token_id| char::from_u32(*token_id).unwrap_or('\u{FFFD}'))

22 - .collect())

23 - }

24 -

25 - fn token_to_id(&self, _token: &str) -> Option<u32> {

26 - None

27 - }

28 -}

29 -

7 #[test]

8 fn factory_contains_and_lists_registered_parsers() {

9 let factory = ReasoningParserFactory::new();

@@ -107,7 +84,7 @@ fn factory_resolves_minimax_m3_before_generic_minimax() {

85 #[test]

86 fn factory_rejects_unknown_parser_names() {

110 - let tokenizer = Arc::new(FakeTokenizer);

87 + let tokenizer = Arc::new(TestTokenizer::new());

88 let factory = ReasoningParserFactory::new();

89 let error = match factory.create("missing", tokenizer) {

90 Ok(_) => panic!("expected parser lookup to fail"),

rust/src/chat/src/parser/unified.rs

+6 -31

@@ -75,39 +75,14 @@ impl UnifiedParserFactory {

75 mod tests {

76 use std::sync::Arc;

78 - use vllm_tokenizer::Tokenizer;

78 + use vllm_tokenizer::test_utils::TestTokenizer;

80 use super::{UnifiedParserFactory, names};

82 - struct FakeTokenizer;

83 -

84 - impl Tokenizer for FakeTokenizer {

85 - fn encode(

86 - &self,

87 - text: &str,

88 - _add_special_tokens: bool,

89 - ) -> vllm_tokenizer::Result<Vec<u32>> {

90 - Ok(text.chars().map(u32::from).collect())

91 - }

92 -

93 - fn decode(

94 - &self,

95 - token_ids: &[u32],

96 - _skip_special_tokens: bool,

97 - ) -> vllm_tokenizer::Result<String> {

98 - Ok(token_ids

99 - .iter()

100 - .map(|token_id| char::from_u32(*token_id).unwrap_or('\u{FFFD}'))

101 - .collect())

102 - }

103 -

104 - fn token_to_id(&self, token: &str) -> Option<u32> {

105 - match token {

106 - "<|channel>" => Some(1),

107 - "<channel|>" => Some(2),

108 - _ => None,

109 - }

110 - }

82 + fn tokenizer() -> TestTokenizer {

83 + TestTokenizer::new()

84 + .with_regular_token("<|channel>", 256)

85 + .with_regular_token("<channel|>", 257)

86 }

88 #[test]

@@ -119,6 +94,6 @@ mod tests {

94 factory.resolve_name_for_model("google/gemma-4-27b-it"),

95 Some(names::GEMMA4)

96 );

122 - factory.create(names::GEMMA4, &[], Arc::new(FakeTokenizer)).unwrap();

97 + factory.create(names::GEMMA4, &[], Arc::new(tokenizer())).unwrap();

98 }

99 }

rust/src/chat/tests/chat.rs

+34 -103

@@ -21,15 +21,17 @@ use vllm_engine_core_client::protocol::{

21 use vllm_engine_core_client::test_utils::{IpcNamespace, spawn_mock_engine_task};

22 use vllm_engine_core_client::{EngineCoreClient, EngineCoreClientConfig};

23 use vllm_llm::Llm;

24 -use vllm_text::tokenizer::{DynTokenizer, Tokenizer};

24 +use vllm_text::tokenizer::DynTokenizer;

25 use vllm_text::{

26 DecodedLogprobs, DecodedPositionLogprobs, DecodedPromptLogprobs, DecodedTokenLogprob, Prompt,

27 TextBackend,

28 };

29 +use vllm_tokenizer::test_utils::TestTokenizer;

30 use zeromq::prelude::{SocketRecv, SocketSend};

31 use zeromq::{DealerSocket, PushSocket, ZmqMessage};

33 const SPECIAL_STOP_TOKEN_ID: u32 = 256;

34 +const UNKNOWN_DECODE_TOKEN_ID: u32 = 10_000;

36 fn request_output(

37 request_id: &str,

@@ -158,45 +160,18 @@ async fn connect_chat_llm_with_ipc(

160 struct FakeChatBackend {

161 has_template: bool,

162 model_id: String,

163 + tokenizer: DynTokenizer,

164 }

165

163 -#[derive(Debug)]

164 -struct FakeChatTokenizer;

165 -

166 -impl Tokenizer for FakeChatTokenizer {

167 - fn encode(&self, text: &str, _add_special_tokens: bool) -> vllm_tokenizer::Result<Vec<u32>> {

168 - Ok(text.bytes().map(u32::from).collect())

169 - }

170 -

171 - fn decode(

172 - &self,

173 - token_ids: &[u32],

174 - skip_special_tokens: bool,

175 - ) -> vllm_tokenizer::Result<String> {

176 - let bytes = token_ids

177 - .iter()

178 - .filter_map(|id| {

179 - if skip_special_tokens && *id == SPECIAL_STOP_TOKEN_ID {

180 - None

181 - } else {

182 - Some(*id as u8)

183 - }

184 - })

185 - .collect::<Vec<_>>();

186 - Ok(String::from_utf8_lossy(&bytes).into_owned())

187 - }

188 -

189 - fn token_to_id(&self, token: &str) -> Option<u32> {

190 - match token {

191 - "<think>" => Some(0xF001),

192 - "</think>" => Some(0xF002),

193 - "<|START_THINKING|>" => Some(0xF003),

194 - "<|END_THINKING|>" => Some(0xF004),

195 - "◁think▷" => Some(0xF005),

196 - "◁/think▷" => Some(0xF006),

197 - _ => None,

198 - }

199 - }

166 +fn fake_chat_tokenizer() -> TestTokenizer {

167 + TestTokenizer::new()

168 + .with_special_token("<stop>", SPECIAL_STOP_TOKEN_ID)

169 + .with_regular_token("<think>", 0xF001)

170 + .with_regular_token("</think>", 0xF002)

171 + .with_regular_token("<|START_THINKING|>", 0xF003)

172 + .with_regular_token("<|END_THINKING|>", 0xF004)

173 + .with_regular_token("◁think▷", 0xF005)

174 + .with_regular_token("◁/think▷", 0xF006)

175 }

176

177 impl fmt::Debug for FakeChatBackend {

@@ -210,6 +185,7 @@ impl FakeChatBackend {

185 Self {

186 has_template: true,

187 model_id: "test-model".to_string(),

188 + tokenizer: Arc::new(fake_chat_tokenizer()),

189 }

190 }

191

@@ -217,6 +193,7 @@ impl FakeChatBackend {

193 Self {

194 has_template: false,

195 model_id: "test-model".to_string(),

196 + tokenizer: Arc::new(fake_chat_tokenizer()),

197 }

198 }

199

@@ -224,13 +201,19 @@ impl FakeChatBackend {

201 Self {

202 has_template: true,

203 model_id: model_id.into(),

204 + tokenizer: Arc::new(fake_chat_tokenizer()),

205 }

206 }

207 +

208 + fn with_tokenizer(mut self, tokenizer: DynTokenizer) -> Self {

209 + self.tokenizer = tokenizer;

210 + self

211 + }

212 }

213

214 impl TextBackend for FakeChatBackend {

215 fn tokenizer(&self) -> DynTokenizer {

233 - Arc::new(FakeChatTokenizer)

216 + Arc::clone(&self.tokenizer)

217 }

218

219 fn model_id(&self) -> &str {

@@ -282,65 +265,6 @@ impl ChatRenderer for FakeChatBackend {

265 }

266 }

267

285 -#[derive(Clone, Debug)]

286 -struct FailingDecodeBackend {

287 - inner: FakeChatBackend,

288 -}

289 -

290 -#[derive(Debug)]

291 -struct FailingDecodeTokenizer;

292 -

293 -impl Tokenizer for FailingDecodeTokenizer {

294 - fn encode(&self, text: &str, add_special_tokens: bool) -> vllm_tokenizer::Result<Vec<u32>> {

295 - FakeChatTokenizer.encode(text, add_special_tokens)

296 - }

297 -

298 - fn decode(

299 - &self,

300 - token_ids: &[u32],

301 - skip_special_tokens: bool,

302 - ) -> vllm_tokenizer::Result<String> {

303 - if token_ids.contains(&(b'i' as u32)) {

304 - return Err(vllm_tokenizer::TokenizerError("decode failed".to_string()));

305 - }

306 - FakeChatTokenizer.decode(token_ids, skip_special_tokens)

307 - }

308 -

309 - fn token_to_id(&self, token: &str) -> Option<u32> {

310 - FakeChatTokenizer.token_to_id(token)

311 - }

312 -}

313 -

314 -impl TextBackend for FailingDecodeBackend {

315 - fn tokenizer(&self) -> DynTokenizer {

316 - Arc::new(FailingDecodeTokenizer)

317 - }

318 -

319 - fn model_id(&self) -> &str {

320 - self.inner.model_id()

321 - }

322 -}

323 -

324 -impl ChatBackend for FailingDecodeBackend {

325 - fn chat_renderer(&self) -> DynChatRenderer {

326 - Arc::new(self.clone())

327 - }

328 -

329 - fn new_chat_output_processor(

330 - &self,

331 - _request: &mut ChatRequest,

332 - _options: NewChatOutputProcessorOptions<'_>,

333 - ) -> vllm_chat::Result<DynChatOutputProcessor> {

334 - Ok(Box::new(DefaultChatOutputProcessor::plain_text_only()))

335 - }

336 -}

337 -

338 -impl ChatRenderer for FailingDecodeBackend {

339 - fn render(&self, request: &ChatRequest) -> vllm_chat::Result<RenderedPrompt> {

340 - self.inner.render(request)

341 - }

342 -}

343 -

268 /// Skip `LogprobsDelta` events that carry only token_ids (no logprobs),

269 /// returning the next semantically interesting event.

270 async fn next_semantic<S>(stream: &mut S) -> Option<Result<ChatEvent, vllm_chat::Error>>

@@ -738,7 +662,12 @@ async fn chat_stream_reports_decode_failure_as_error_event() {

662 send_outputs(

663 push,

664 EngineCoreOutputs {

741 - outputs: vec![request_output("chat-4", vec![b'i' as u32], None, None)],

665 + outputs: vec![request_output(

666 + "chat-4",

667 + vec![UNKNOWN_DECODE_TOKEN_ID],

668 + None,

669 + None,

670 + )],

671 ..Default::default()

672 },

673 )

@@ -747,9 +676,8 @@ async fn chat_stream_reports_decode_failure_as_error_event() {

676 },

677 );

678

750 - let backend: Arc<dyn ChatTextBackend> = Arc::new(FailingDecodeBackend {

751 - inner: FakeChatBackend::new(),

752 - });

679 + let backend: Arc<dyn ChatTextBackend> =

680 + Arc::new(FakeChatBackend::new().with_tokenizer(Arc::new(TestTokenizer::new())));

681 let chat = connect_chat_llm_with_ipc(

682 EngineCoreClientConfig::new_single(handshake_address),

683 &ipc,

@@ -769,7 +697,10 @@ async fn chat_stream_reports_decode_failure_as_error_event() {

697

698 match timeout(Duration::from_secs(2), stream.next()).await.unwrap() {

699 Some(Err(vllm_chat::Error::Text(vllm_text::Error::Tokenizer(message)))) => {

772 - assert_eq!(message, "decode failed");

700 + assert_eq!(

701 + message,

702 + format!("test tokenizer cannot decode unknown token id {UNKNOWN_DECODE_TOKEN_ID}")

703 + );

704 }

705 other => panic!("unexpected event after close: {other:?}"),

706 }

rust/src/parser/Cargo.toml

@@ -23,6 +23,7 @@ expect-test.workspace = true

23 futures.workspace = true

24 openai-protocol.workspace = true

25 tool-parser.workspace = true

26 +vllm-tokenizer = { workspace = true, features = ["test-utils"] }

28 [[bench]]

29 name = "deepseek_v3"

rust/src/parser/benches/utils/adapter.rs

@@ -27,6 +27,10 @@ impl Tokenizer for BenchTokenizer {

27 fn token_to_id(&self, _token: &str) -> Option<u32> {

28 Some(u32::MAX)

29 }

30 +

31 + fn id_to_token(&self, _id: u32) -> Option<String> {

32 + Some("\u{FFFD}".to_string())

33 + }

34 }

36 /// Bench-only adapter that exposes a unified parser through the tool-parser

rust/src/parser/src/reasoning/seed_oss.rs

+15 -12

@@ -49,11 +49,14 @@ mod tests {

49 use std::sync::Arc;

51 use super::SeedOssReasoningParser;

52 - use crate::reasoning::{ReasoningParser, tests::FakeTokenizer};

52 + use crate::reasoning::{

53 + ReasoningParser,

54 + tests::{SEED_THINK_END_ID, SEED_THINK_START_ID, fake_tokenizer},

55 + };

57 #[test]

58 fn without_prompt_markers_expects_start_token() {

56 - let tokenizer = Arc::new(FakeTokenizer);

59 + let tokenizer = Arc::new(fake_tokenizer());

60 let mut parser = SeedOssReasoningParser::new(tokenizer).unwrap();

62 let delta = parser.push("implicit reasoning</seed:think>answer").unwrap();

@@ -66,10 +69,10 @@ mod tests {

70 #[test]

71 fn picks_up_prompt_start_boundary() {

69 - let tokenizer = Arc::new(FakeTokenizer);

72 + let tokenizer = Arc::new(fake_tokenizer());

73 let mut parser = SeedOssReasoningParser::new(tokenizer).unwrap();

71 - // Prompt prefills `<seed:think>` (id 10), opening reasoning before the stream.

72 - parser.initialize(&[10]).unwrap();

74 + // Prompt prefills `<seed:think>`, opening reasoning before the stream.

75 + parser.initialize(&[SEED_THINK_START_ID]).unwrap();

77 let delta = parser.push("reason</seed:think>answer").unwrap();

78 assert_eq!(delta.reasoning.as_deref(), Some("reason"));

@@ -78,10 +81,10 @@ mod tests {

82 #[test]

83 fn respects_prompt_end_boundary() {

81 - let tokenizer = Arc::new(FakeTokenizer);

84 + let tokenizer = Arc::new(fake_tokenizer());

85 let mut parser = SeedOssReasoningParser::new(tokenizer).unwrap();

83 - // Prompt already closed reasoning with `</seed:think>` (id 11).

84 - parser.initialize(&[11]).unwrap();

86 + // Prompt already closed reasoning with `</seed:think>`.

87 + parser.initialize(&[SEED_THINK_END_ID]).unwrap();

89 let delta = parser.push("answer").unwrap();

90 assert_eq!(delta.reasoning, None);

@@ -91,7 +94,7 @@ mod tests {

94 #[test]

95 fn handles_explicit_start_token() {

96 // An explicit start delimiter must not leak into reasoning text.

94 - let tokenizer = Arc::new(FakeTokenizer);

97 + let tokenizer = Arc::new(fake_tokenizer());

98 let mut parser = SeedOssReasoningParser::new(tokenizer).unwrap();

100 let delta = parser.push("<seed:think>reason</seed:think>answer").unwrap();

@@ -103,7 +106,7 @@ mod tests {

106 fn streams_explicit_start_token_across_pushes() {

107 // Start token, reasoning body, end token, and content arrive in separate

108 // streaming deltas.

106 - let tokenizer = Arc::new(FakeTokenizer);

109 + let tokenizer = Arc::new(fake_tokenizer());

110 let mut parser = SeedOssReasoningParser::new(tokenizer).unwrap();

111

112 let mut reasoning = String::new();

@@ -131,9 +134,9 @@ mod tests {

134

135 #[test]

136 fn handles_partial_delimiters_across_pushes() {

134 - let tokenizer = Arc::new(FakeTokenizer);

137 + let tokenizer = Arc::new(fake_tokenizer());

138 let mut parser = SeedOssReasoningParser::new(tokenizer).unwrap();

136 - parser.initialize(&[10]).unwrap();

139 + parser.initialize(&[SEED_THINK_START_ID]).unwrap();

140

141 // Closing delimiter `</seed:think>` arrives in two halves.

142 let first = parser.push("reason</seed:").unwrap();

rust/src/parser/src/reasoning/step3p5.rs

+20 -17

@@ -127,14 +127,17 @@ mod tests {

127 use std::sync::Arc;

128

129 use super::Step3p5ReasoningParser;

130 - use crate::reasoning::{ReasoningParser, tests::FakeTokenizer};

130 + use crate::reasoning::{

131 + ReasoningParser,

132 + tests::{THINK_START_ID, fake_tokenizer},

133 + };

134

135 #[test]

136 fn picks_up_prompt_start_boundary() {

134 - let tokenizer = Arc::new(FakeTokenizer);

137 + let tokenizer = Arc::new(fake_tokenizer());

138 let mut parser = Step3p5ReasoningParser::new(tokenizer).unwrap();

136 - // Prompt prefills `<think>` (id 1), opening reasoning before the stream.

137 - parser.initialize(&[1]).unwrap();

139 + // Prompt prefills `<think>`, opening reasoning before the stream.

140 + parser.initialize(&[THINK_START_ID]).unwrap();

141

142 let delta = parser.push("This is a reasoning section</think>This is the rest").unwrap();

143 assert_eq!(

@@ -146,7 +149,7 @@ mod tests {

149

150 #[test]

151 fn handles_unterminated_reasoning() {

149 - let tokenizer = Arc::new(FakeTokenizer);

152 + let tokenizer = Arc::new(fake_tokenizer());

153 let mut parser = Step3p5ReasoningParser::new(tokenizer).unwrap();

154

155 let pushed = parser.push("<think>reason without end").unwrap();

@@ -159,7 +162,7 @@ mod tests {

162

163 #[test]

164 fn handles_empty_input() {

162 - let tokenizer = Arc::new(FakeTokenizer);

165 + let tokenizer = Arc::new(fake_tokenizer());

166 let mut parser = Step3p5ReasoningParser::new(tokenizer).unwrap();

167

168 let pushed = parser.push("").unwrap();

@@ -172,9 +175,9 @@ mod tests {

175 fn complex_newline_pattern_trims_only_single_framing_newline_each_side() {

176 // Only the immediately-adjacent framing `\n` is dropped on each side of

177 // `</think>`; surrounding newlines remain part of reasoning/content.

175 - let tokenizer = Arc::new(FakeTokenizer);

178 + let tokenizer = Arc::new(fake_tokenizer());

179 let mut parser = Step3p5ReasoningParser::new(tokenizer).unwrap();

177 - parser.initialize(&[1]).unwrap();

180 + parser.initialize(&[THINK_START_ID]).unwrap();

181

182 let delta = parser

183 .push("\n This is a \n reasoning section\n\n\n</think>\n\nThis is the rest")

@@ -188,7 +191,7 @@ mod tests {

191

192 #[test]

193 fn drops_framing_newlines_in_single_push() {

191 - let tokenizer = Arc::new(FakeTokenizer);

194 + let tokenizer = Arc::new(fake_tokenizer());

195 let mut parser = Step3p5ReasoningParser::new(tokenizer).unwrap();

196

197 let delta = parser.push("<think>reason\n</think>\nanswer").unwrap();

@@ -198,7 +201,7 @@ mod tests {

201

202 #[test]

203 fn drops_framing_newlines_across_pushes() {

201 - let tokenizer = Arc::new(FakeTokenizer);

204 + let tokenizer = Arc::new(fake_tokenizer());

205 let mut parser = Step3p5ReasoningParser::new(tokenizer).unwrap();

206

207 // The trailing `\n` from the first push is held until we know whether

@@ -219,7 +222,7 @@ mod tests {

222

223 #[test]

224 fn replays_held_newline_when_more_reasoning_follows() {

222 - let tokenizer = Arc::new(FakeTokenizer);

225 + let tokenizer = Arc::new(fake_tokenizer());

226 let mut parser = Step3p5ReasoningParser::new(tokenizer).unwrap();

227

228 let first = parser.push("<think>reason\n").unwrap();

@@ -232,7 +235,7 @@ mod tests {

235

236 #[test]

237 fn finish_flushes_held_newline_in_unterminated_stream() {

235 - let tokenizer = Arc::new(FakeTokenizer);

238 + let tokenizer = Arc::new(fake_tokenizer());

239 let mut parser = Step3p5ReasoningParser::new(tokenizer).unwrap();

240

241 let first = parser.push("<think>reason\n").unwrap();

@@ -245,7 +248,7 @@ mod tests {

248

249 #[test]

250 fn preserves_inner_newlines_in_reasoning() {

248 - let tokenizer = Arc::new(FakeTokenizer);

251 + let tokenizer = Arc::new(fake_tokenizer());

252 let mut parser = Step3p5ReasoningParser::new(tokenizer).unwrap();

253

254 let delta = parser.push("<think>line1\nline2</think>tail").unwrap();

@@ -257,7 +260,7 @@ mod tests {

260 fn trims_only_one_trailing_reasoning_newline() {

261 // Only the single framing newline immediately before `</think>` is

262 // dropped; earlier newlines in the reasoning body are preserved.

260 - let tokenizer = Arc::new(FakeTokenizer);

263 + let tokenizer = Arc::new(fake_tokenizer());

264 let mut parser = Step3p5ReasoningParser::new(tokenizer).unwrap();

265

266 let delta = parser.push("<think>reason\n\n</think>answer").unwrap();

@@ -269,7 +272,7 @@ mod tests {

272 fn drops_only_first_content_newline_after_transition() {

273 // The leading-`\n` drop applies only to the first content delta after

274 // `</think>`; later deltas pass through untouched.

272 - let tokenizer = Arc::new(FakeTokenizer);

275 + let tokenizer = Arc::new(fake_tokenizer());

276 let mut parser = Step3p5ReasoningParser::new(tokenizer).unwrap();

277

278 let first = parser.push("<think>reason</think>").unwrap();

@@ -288,7 +291,7 @@ mod tests {

291

292 #[test]

293 fn passes_through_clean_boundary_without_framing_newlines() {

291 - let tokenizer = Arc::new(FakeTokenizer);

294 + let tokenizer = Arc::new(fake_tokenizer());

295 let mut parser = Step3p5ReasoningParser::new(tokenizer).unwrap();

296

297 let delta = parser.push("<think>reason</think>tail").unwrap();

@@ -298,7 +301,7 @@ mod tests {

301

302 #[test]

303 fn handles_empty_reasoning_section() {

301 - let tokenizer = Arc::new(FakeTokenizer);

304 + let tokenizer = Arc::new(fake_tokenizer());

305 let mut parser = Step3p5ReasoningParser::new(tokenizer).unwrap();

306

307 let delta = parser.push("<think></think>answer").unwrap();

rust/src/parser/src/reasoning/tests.rs

+49 -61

@@ -1,54 +1,42 @@

1 use std::sync::Arc;

3 -use vllm_tokenizer::Tokenizer;

3 +use vllm_tokenizer::test_utils::TestTokenizer;

5 use super::{

6 DeepSeekR1ReasoningParser, DelimitedReasoningParser, MiniMaxM3ReasoningParser,

7 Qwen3ReasoningParser, ReasoningParser,

8 };

10 -pub(crate) struct FakeTokenizer;

11 -

12 -impl Tokenizer for FakeTokenizer {

13 - fn encode(&self, text: &str, _add_special_tokens: bool) -> vllm_tokenizer::Result<Vec<u32>> {

14 - Ok(text.chars().map(u32::from).collect())

15 - }

16 -

17 - fn decode(

18 - &self,

19 - token_ids: &[u32],

20 - _skip_special_tokens: bool,

21 - ) -> vllm_tokenizer::Result<String> {

22 - Ok(token_ids

23 - .iter()

24 - .map(|token_id| char::from_u32(*token_id).unwrap_or('\u{FFFD}'))

25 - .collect())

26 - }

27 -

28 - fn token_to_id(&self, token: &str) -> Option<u32> {

29 - match token {

30 - "<think>" => Some(1),

31 - "</think>" => Some(2),

32 - "<|START_THINKING|>" => Some(3),

33 - "<|END_THINKING|>" => Some(4),

34 - "◁think▷" => Some(5),

35 - "◁/think▷" => Some(6),

36 - "<mm:think>" => Some(8),

37 - "</mm:think>" => Some(9),

38 - "<seed:think>" => Some(10),

39 - "</seed:think>" => Some(11),

40 - _ => None,

41 - }

42 - }

43 -

44 - fn is_special_id(&self, token_id: u32) -> bool {

45 - token_id == 7

46 - }

10 +pub(crate) const THINK_START_ID: u32 = 256;

11 +pub(crate) const THINK_END_ID: u32 = 257;

12 +pub(crate) const START_THINKING_ID: u32 = 258;

13 +pub(crate) const END_THINKING_ID: u32 = 259;

14 +pub(crate) const MINIMAX_THINK_START_ID: u32 = 260;

15 +pub(crate) const MINIMAX_THINK_END_ID: u32 = 261;

16 +pub(crate) const SPECIAL_BOUNDARY_ID: u32 = 262;

17 +pub(crate) const MM_THINK_START_ID: u32 = 263;

18 +pub(crate) const MM_THINK_END_ID: u32 = 264;

19 +pub(crate) const SEED_THINK_START_ID: u32 = 265;

20 +pub(crate) const SEED_THINK_END_ID: u32 = 266;

21 +

22 +pub(crate) fn fake_tokenizer() -> TestTokenizer {

23 + TestTokenizer::new()

24 + .with_regular_token("<think>", THINK_START_ID)

25 + .with_regular_token("</think>", THINK_END_ID)

26 + .with_regular_token("<|START_THINKING|>", START_THINKING_ID)

27 + .with_regular_token("<|END_THINKING|>", END_THINKING_ID)

28 + .with_regular_token("◁think▷", MINIMAX_THINK_START_ID)

29 + .with_regular_token("◁/think▷", MINIMAX_THINK_END_ID)

30 + .with_special_token("<special-boundary>", SPECIAL_BOUNDARY_ID)

31 + .with_regular_token("<mm:think>", MM_THINK_START_ID)

32 + .with_regular_token("</mm:think>", MM_THINK_END_ID)

33 + .with_regular_token("<seed:think>", SEED_THINK_START_ID)

34 + .with_regular_token("</seed:think>", SEED_THINK_END_ID)

35 }

37 #[test]

38 fn delimited_content_only_stream() {

51 - let tokenizer = Arc::new(FakeTokenizer);

39 + let tokenizer = Arc::new(fake_tokenizer());

40 let mut parser =

41 DelimitedReasoningParser::new(tokenizer, "<think>", "</think>", false).unwrap();

@@ -60,7 +48,7 @@ fn delimited_content_only_stream() {

49 #[test]

50 fn delimited_single_chunk_with_reasoning_and_content() {

63 - let tokenizer = Arc::new(FakeTokenizer);

51 + let tokenizer = Arc::new(fake_tokenizer());

52 let mut parser =

53 DelimitedReasoningParser::new(tokenizer, "<think>", "</think>", false).unwrap();

@@ -71,7 +59,7 @@ fn delimited_single_chunk_with_reasoning_and_content() {

60 #[test]

61 fn delimited_partial_tokens_across_chunks() {

74 - let tokenizer = Arc::new(FakeTokenizer);

62 + let tokenizer = Arc::new(fake_tokenizer());

63 let mut parser =

64 DelimitedReasoningParser::new(tokenizer, "<think>", "</think>", false).unwrap();

@@ -83,10 +71,10 @@ fn delimited_partial_tokens_across_chunks() {

72 #[test]

73 fn delimited_finish_flushes_buffer() {

86 - let tokenizer = Arc::new(FakeTokenizer);

74 + let tokenizer = Arc::new(fake_tokenizer());

75 let mut parser =

76 DelimitedReasoningParser::new(tokenizer, "<think>", "</think>", false).unwrap();

89 - parser.initialize(&[1]);

77 + parser.initialize(&[THINK_START_ID]);

79 let delta = parser.push("unfinished</thi");

80 assert_eq!(delta.reasoning.as_deref(), Some("unfinished"));

@@ -96,7 +84,7 @@ fn delimited_finish_flushes_buffer() {

85 #[test]

86 fn qwen3_without_prompt_markers_expects_start_token() {

99 - let tokenizer = Arc::new(FakeTokenizer);

87 + let tokenizer = Arc::new(fake_tokenizer());

88 let mut parser = Qwen3ReasoningParser::new(tokenizer).unwrap();

90 let delta = parser.push("reason</think>answer").unwrap();

@@ -106,9 +94,9 @@ fn qwen3_without_prompt_markers_expects_start_token() {

95 #[test]

96 fn qwen3_prompt_end_marker_starts_in_content() {

109 - let tokenizer = Arc::new(FakeTokenizer);

97 + let tokenizer = Arc::new(fake_tokenizer());

98 let mut parser = Qwen3ReasoningParser::new(tokenizer).unwrap();

111 - parser.initialize(&[2]).unwrap();

99 + parser.initialize(&[THINK_END_ID]).unwrap();

100

101 let delta = parser.push("answer").unwrap();

102 assert_eq!(delta.reasoning, None);

@@ -117,7 +105,7 @@ fn qwen3_prompt_end_marker_starts_in_content() {

105

106 #[test]

107 fn qwen3_tolerates_old_and_new_formats() {

120 - let tokenizer = Arc::new(FakeTokenizer);

108 + let tokenizer = Arc::new(fake_tokenizer());

109

110 let mut old_parser = Qwen3ReasoningParser::new(tokenizer.clone()).unwrap();

111 let old = old_parser.push("<think>reason</think>answer").unwrap();

@@ -125,7 +113,7 @@ fn qwen3_tolerates_old_and_new_formats() {

113 assert_eq!(old.content.as_deref(), Some("answer"));

114

115 let mut new_parser = Qwen3ReasoningParser::new(tokenizer).unwrap();

128 - new_parser.initialize(&[1]).unwrap();

116 + new_parser.initialize(&[THINK_START_ID]).unwrap();

117 let new = new_parser.push("reason</think>answer").unwrap();

118 assert_eq!(new.reasoning.as_deref(), Some("reason"));

119 assert_eq!(new.content.as_deref(), Some("answer"));

@@ -133,10 +121,10 @@ fn qwen3_tolerates_old_and_new_formats() {

121

122 #[test]

123 fn qwen3_stops_scanning_at_last_special_token() {

136 - let tokenizer = Arc::new(FakeTokenizer);

124 + let tokenizer = Arc::new(fake_tokenizer());

125 let mut parser = Qwen3ReasoningParser::new(tokenizer).unwrap();

126

139 - parser.initialize(&[1, 7]).unwrap();

127 + parser.initialize(&[THINK_START_ID, SPECIAL_BOUNDARY_ID]).unwrap();

128

129 let delta = parser.push("answer").unwrap();

130 assert_eq!(delta.reasoning, None);

@@ -145,7 +133,7 @@ fn qwen3_stops_scanning_at_last_special_token() {

133

134 #[test]

135 fn deepseek_r1_defaults_to_reasoning_without_prompt_boundary() {

148 - let tokenizer = Arc::new(FakeTokenizer);

136 + let tokenizer = Arc::new(fake_tokenizer());

137 let mut parser = DeepSeekR1ReasoningParser::new(tokenizer).unwrap();

138

139 let delta = parser.push("reason</think>answer").unwrap();

@@ -155,10 +143,10 @@ fn deepseek_r1_defaults_to_reasoning_without_prompt_boundary() {

143

144 #[test]

145 fn deepseek_r1_stops_scanning_at_last_special_token() {

158 - let tokenizer = Arc::new(FakeTokenizer);

146 + let tokenizer = Arc::new(fake_tokenizer());

147 let mut parser = DeepSeekR1ReasoningParser::new(tokenizer).unwrap();

148

161 - parser.initialize(&[2, 7]).unwrap();

149 + parser.initialize(&[THINK_END_ID, SPECIAL_BOUNDARY_ID]).unwrap();

150

151 let delta = parser.push("reason</think>answer").unwrap();

152 assert_eq!(delta.reasoning.as_deref(), Some("reason"));

@@ -167,7 +155,7 @@ fn deepseek_r1_stops_scanning_at_last_special_token() {

155

156 #[test]

157 fn minimax_m3_handles_explicit_think_delimiters() {

170 - let tokenizer = Arc::new(FakeTokenizer);

158 + let tokenizer = Arc::new(fake_tokenizer());

159 let mut parser = MiniMaxM3ReasoningParser::new(tokenizer).unwrap();

160

161 let delta = parser.push("<mm:think>reason</mm:think>answer").unwrap();

@@ -177,7 +165,7 @@ fn minimax_m3_handles_explicit_think_delimiters() {

165

166 #[test]

167 fn minimax_m3_drops_leading_end_marker() {

180 - let tokenizer = Arc::new(FakeTokenizer);

168 + let tokenizer = Arc::new(fake_tokenizer());

169 let mut parser = MiniMaxM3ReasoningParser::new(tokenizer).unwrap();

170

171 let delta = parser.push("</mm:think>answer").unwrap();

@@ -187,7 +175,7 @@ fn minimax_m3_drops_leading_end_marker() {

175

176 #[test]

177 fn minimax_m3_preserves_non_leading_end_marker() {

190 - let tokenizer = Arc::new(FakeTokenizer);

178 + let tokenizer = Arc::new(fake_tokenizer());

179 let mut parser = MiniMaxM3ReasoningParser::new(tokenizer).unwrap();

180

181 let delta = parser.push("XXX</mm:think>YYY").unwrap();

@@ -197,7 +185,7 @@ fn minimax_m3_preserves_non_leading_end_marker() {

185

186 #[test]

187 fn minimax_m3_drops_split_leading_end_marker() {

200 - let tokenizer = Arc::new(FakeTokenizer);

188 + let tokenizer = Arc::new(fake_tokenizer());

189 let mut parser = MiniMaxM3ReasoningParser::new(tokenizer).unwrap();

190

191 assert!(parser.push("</mm").unwrap().is_empty());

@@ -208,9 +196,9 @@ fn minimax_m3_drops_split_leading_end_marker() {

196

197 #[test]

198 fn minimax_m3_uses_prompt_prefilled_start_marker() {

211 - let tokenizer = Arc::new(FakeTokenizer);

199 + let tokenizer = Arc::new(fake_tokenizer());

200 let mut parser = MiniMaxM3ReasoningParser::new(tokenizer).unwrap();

213 - parser.initialize(&[8]).unwrap();

201 + parser.initialize(&[MM_THINK_START_ID]).unwrap();

202

203 let delta = parser.push("reason</mm:think>answer").unwrap();

204 assert_eq!(delta.reasoning.as_deref(), Some("reason"));

@@ -219,9 +207,9 @@ fn minimax_m3_uses_prompt_prefilled_start_marker() {

207

208 #[test]

209 fn minimax_m3_uses_prompt_prefilled_end_marker() {

222 - let tokenizer = Arc::new(FakeTokenizer);

210 + let tokenizer = Arc::new(fake_tokenizer());

211 let mut parser = MiniMaxM3ReasoningParser::new(tokenizer).unwrap();

224 - parser.initialize(&[9]).unwrap();

212 + parser.initialize(&[MM_THINK_END_ID]).unwrap();

213

214 let delta = parser.push("answer").unwrap();

215 assert_eq!(delta.reasoning, None);

rust/src/parser/src/unified/combined.rs

+6 -31

@@ -124,42 +124,17 @@ impl UnifiedParser for CombinedParser {

124 mod tests {

125 use std::sync::Arc;

126

127 - use vllm_tokenizer::Tokenizer;

127 + use vllm_tokenizer::test_utils::TestTokenizer;

128

129 use super::CombinedParser;

130 use crate::reasoning::{Qwen3ReasoningParser, ReasoningDelta, ReasoningParser};

131 use crate::tool::{Qwen3XmlToolParser, Tool, ToolParser};

132 use crate::unified::{UnifiedParser, UnifiedParserEvent, UnifiedParserOutput};

133

134 - struct FakeTokenizer;

135 -

136 - impl Tokenizer for FakeTokenizer {

137 - fn encode(

138 - &self,

139 - text: &str,

140 - _add_special_tokens: bool,

141 - ) -> vllm_tokenizer::Result<Vec<u32>> {

142 - Ok(text.chars().map(u32::from).collect())

143 - }

144 -

145 - fn decode(

146 - &self,

147 - token_ids: &[u32],

148 - _skip_special_tokens: bool,

149 - ) -> vllm_tokenizer::Result<String> {

150 - Ok(token_ids

151 - .iter()

152 - .map(|token_id| char::from_u32(*token_id).unwrap_or('\u{FFFD}'))

153 - .collect())

154 - }

155 -

156 - fn token_to_id(&self, token: &str) -> Option<u32> {

157 - match token {

158 - "<think>" => Some(1),

159 - "</think>" => Some(2),

160 - _ => None,

161 - }

162 - }

134 + fn tokenizer() -> TestTokenizer {

135 + TestTokenizer::new()

136 + .with_regular_token("<think>", 256)

137 + .with_regular_token("</think>", 257)

138 }

139

140 fn test_tools() -> Vec<Tool> {

@@ -273,7 +248,7 @@ mod tests {

248

249 #[test]

250 fn combined_parser_emits_reasoning_and_text() {

276 - let tokenizer = Arc::new(FakeTokenizer);

251 + let tokenizer = Arc::new(tokenizer());

252 let reasoning = Qwen3ReasoningParser::create(tokenizer).unwrap();

253 let mut parser = CombinedParser::new(Some(reasoning), None);

254

rust/src/parser/src/unified/gemma4.rs

+15 -66

@@ -515,7 +515,7 @@ mod tests {

515

516 use serde_json::{Value, json};

517 use thiserror_ext::AsReport;

518 - use vllm_tokenizer::Tokenizer;

518 + use vllm_tokenizer::test_utils::TestTokenizer;

519 use winnow::combinator::{eof, terminated};

520 use winnow::error::ErrMode;

521 use winnow::prelude::*;

@@ -527,66 +527,15 @@ mod tests {

527 use crate::tool::Tool;

528 use crate::unified::{UnifiedParserEvent, parsing_failed};

529

530 - struct FakeTokenizer;

530 + const CHANNEL_START_ID: u32 = 256;

531 + const CHANNEL_END_ID: u32 = 257;

532 + const TURN_BOUNDARY_ID: u32 = 258;

533

532 - impl Tokenizer for FakeTokenizer {

533 - fn encode(

534 - &self,

535 - text: &str,

536 - _add_special_tokens: bool,

537 - ) -> vllm_tokenizer::Result<Vec<u32>> {

538 - Ok(text.chars().map(u32::from).collect())

539 - }

540 -

541 - fn decode(

542 - &self,

543 - token_ids: &[u32],

544 - _skip_special_tokens: bool,

545 - ) -> vllm_tokenizer::Result<String> {

546 - Ok(token_ids

547 - .iter()

548 - .map(|token_id| char::from_u32(*token_id).unwrap_or('\u{FFFD}'))

549 - .collect())

550 - }

551 -

552 - fn token_to_id(&self, token: &str) -> Option<u32> {

553 - match token {

554 - CHANNEL_START => Some(100),

555 - CHANNEL_END => Some(101),

556 - _ => None,

557 - }

558 - }

559 -

560 - fn is_special_id(&self, token_id: u32) -> bool {

561 - matches!(token_id, 100..=105)

562 - }

563 - }

564 -

565 - struct MissingTokenTokenizer;

566 -

567 - impl Tokenizer for MissingTokenTokenizer {

568 - fn encode(

569 - &self,

570 - text: &str,

571 - _add_special_tokens: bool,

572 - ) -> vllm_tokenizer::Result<Vec<u32>> {

573 - Ok(text.chars().map(u32::from).collect())

574 - }

575 -

576 - fn decode(

577 - &self,

578 - token_ids: &[u32],

579 - _skip_special_tokens: bool,

580 - ) -> vllm_tokenizer::Result<String> {

581 - Ok(token_ids

582 - .iter()

583 - .map(|token_id| char::from_u32(*token_id).unwrap_or('\u{FFFD}'))

584 - .collect())

585 - }

586 -

587 - fn token_to_id(&self, _token: &str) -> Option<u32> {

588 - None

589 - }

534 + fn tokenizer() -> TestTokenizer {

535 + TestTokenizer::new()

536 + .with_special_token(CHANNEL_START, CHANNEL_START_ID)

537 + .with_special_token(CHANNEL_END, CHANNEL_END_ID)

538 + .with_special_token("<turn-boundary>", TURN_BOUNDARY_ID)

539 }

540

541 trait UnifiedParserTestExt {

@@ -716,12 +665,12 @@ mod tests {

665 }

666

667 fn test_parser() -> Gemma4UnifiedParser {

719 - Gemma4UnifiedParser::new(&test_tools(), Arc::new(FakeTokenizer)).unwrap()

668 + Gemma4UnifiedParser::new(&test_tools(), Arc::new(tokenizer())).unwrap()

669 }

670

671 #[test]

672 fn gemma4_create_requires_channel_start_token() {

724 - let error = match Gemma4UnifiedParser::new(&test_tools(), Arc::new(MissingTokenTokenizer)) {

673 + let error = match Gemma4UnifiedParser::new(&test_tools(), Arc::new(TestTokenizer::new())) {

674 Ok(_) => panic!("expected missing token error"),

675 Err(error) => error,

676 };

@@ -1046,7 +995,7 @@ mod tests {

995 #[test]

996 fn gemma4_initialize_open_channel_prompt_starts_in_reasoning() {

997 let mut parser = test_parser();

1049 - parser.initialize(&[100, 3000, 3001]).unwrap();

998 + parser.initialize(&[CHANNEL_START_ID, 3000, 3001]).unwrap();

999

1000 let output = parser.parse_complete("reason<channel|>answer").unwrap();

1001

@@ -1057,7 +1006,7 @@ mod tests {

1006 #[test]

1007 fn gemma4_initialize_turn_prompt_starts_in_text() {

1008 let mut parser = test_parser();

1060 - parser.initialize(&[104, 3000, 3001]).unwrap();

1009 + parser.initialize(&[TURN_BOUNDARY_ID, 3000, 3001]).unwrap();

1010

1011 let output = parser.parse_complete("<|channel>thought\nreason<channel|>answer").unwrap();

1012

@@ -1068,7 +1017,7 @@ mod tests {

1017 #[test]

1018 fn gemma4_initialize_special_token_caps_boundary_scan() {

1019 let mut parser = test_parser();

1071 - parser.initialize(&[100, 3000, 104, 3001]).unwrap();

1020 + parser.initialize(&[CHANNEL_START_ID, 3000, TURN_BOUNDARY_ID, 3001]).unwrap();

1021

1022 let output = parser.parse_complete("answer").unwrap();

1023

@@ -1079,7 +1028,7 @@ mod tests {

1028 #[test]

1029 fn gemma4_initialize_closed_channel_prompt_starts_in_text() {

1030 let mut parser = test_parser();

1082 - parser.initialize(&[100, 3000, 3001, 101]).unwrap();

1031 + parser.initialize(&[CHANNEL_START_ID, 3000, 3001, CHANNEL_END_ID]).unwrap();

1032

1033 let output = parser.parse_complete("answer").unwrap();

1034

rust/src/server/Cargo.toml

@@ -64,6 +64,7 @@ tempfile.workspace = true

64 tokio = { workspace = true, features = ["test-util"] }

65 tower.workspace = true

66 vllm-engine-core-client = { workspace = true, features = ["test-util"] }

67 +vllm-tokenizer = { workspace = true, features = ["test-utils"] }

68 zeromq.workspace = true

70 [lints]

rust/src/server/src/grpc/tests.rs

+3 -30

@@ -24,8 +24,9 @@ use vllm_engine_core_client::protocol::{

24 use vllm_engine_core_client::test_utils::{IpcNamespace, spawn_mock_engine_task};

25 use vllm_engine_core_client::{EngineCoreClient, EngineCoreClientConfig, EngineId};

26 use vllm_llm::Llm;

27 -use vllm_text::tokenizer::{DynTokenizer, Tokenizer};

27 +use vllm_text::tokenizer::DynTokenizer;

28 use vllm_text::{Prompt, TextBackend};

29 +use vllm_tokenizer::test_utils::TestTokenizer;

30 use zeromq::prelude::{SocketRecv, SocketSend};

31 use zeromq::{DealerSocket, PushSocket, ZmqMessage};

@@ -155,37 +156,9 @@ fn test_llm(client: EngineCoreClient) -> Llm {

156 #[derive(Clone, Debug)]

157 struct FakeTextBackend;

158

158 -#[derive(Debug)]

159 -struct FakeTokenizer;

160 -

161 -impl Tokenizer for FakeTokenizer {

162 - fn encode(

163 - &self,

164 - text: &str,

165 - _add_special_tokens: bool,

166 - ) -> vllm_text::tokenizer::Result<Vec<u32>> {

167 - Ok(text.bytes().map(u32::from).collect())

168 - }

169 -

170 - fn decode(

171 - &self,

172 - token_ids: &[u32],

173 - _skip_special_tokens: bool,

174 - ) -> vllm_text::tokenizer::Result<String> {

175 - Ok(

176 - String::from_utf8_lossy(&token_ids.iter().map(|id| *id as u8).collect::<Vec<_>>())

177 - .into_owned(),

178 - )

179 - }

180 -

181 - fn token_to_id(&self, token: &str) -> Option<u32> {

182 - token.bytes().next().map(u32::from)

183 - }

184 -}

185 -

159 impl TextBackend for FakeTextBackend {

160 fn tokenizer(&self) -> DynTokenizer {

188 - Arc::new(FakeTokenizer)

161 + Arc::new(TestTokenizer::new())

162 }

163

164 fn model_id(&self) -> &str {

rust/src/server/src/routes/http_client_tests.rs

+3 -30

@@ -24,8 +24,9 @@ use vllm_engine_core_client::protocol::{

24 use vllm_engine_core_client::test_utils::{IpcNamespace, spawn_mock_engine_task};

25 use vllm_engine_core_client::{EngineCoreClient, EngineCoreClientConfig, EngineId};

26 use vllm_llm::Llm;

27 -use vllm_text::tokenizer::{DynTokenizer, Tokenizer};

27 +use vllm_text::tokenizer::DynTokenizer;

28 use vllm_text::{Prompt, TextBackend};

29 +use vllm_tokenizer::test_utils::TestTokenizer;

30 use zeromq::prelude::{SocketRecv, SocketSend};

31 use zeromq::{DealerSocket, PushSocket, ZmqMessage};

@@ -151,37 +152,9 @@ fn test_llm(client: EngineCoreClient) -> Llm {

152 #[derive(Clone, Debug)]

153 struct FakeChatBackend;

154

154 -#[derive(Debug)]

155 -struct FakeChatTokenizer;

156 -

157 -impl Tokenizer for FakeChatTokenizer {

158 - fn encode(

159 - &self,

160 - text: &str,

161 - _add_special_tokens: bool,

162 - ) -> vllm_text::tokenizer::Result<Vec<u32>> {

163 - Ok(text.bytes().map(u32::from).collect())

164 - }

165 -

166 - fn decode(

167 - &self,

168 - token_ids: &[u32],

169 - _skip_special_tokens: bool,

170 - ) -> vllm_text::tokenizer::Result<String> {

171 - Ok(

172 - String::from_utf8_lossy(&token_ids.iter().map(|id| *id as u8).collect::<Vec<_>>())

173 - .into_owned(),

174 - )

175 - }

176 -

177 - fn token_to_id(&self, token: &str) -> Option<u32> {

178 - token.bytes().next().map(u32::from)

179 - }

180 -}

181 -

155 impl TextBackend for FakeChatBackend {

156 fn tokenizer(&self) -> DynTokenizer {

184 - Arc::new(FakeChatTokenizer)

157 + Arc::new(TestTokenizer::new())

158 }

159

160 fn model_id(&self) -> &str {

rust/src/server/src/routes/openai/completions/convert.rs

+15 -39

@@ -194,7 +194,7 @@ mod tests {

194 use axum::http::HeaderMap;

195 use serde_json::json;

196 use vllm_text::Prompt;

197 - use vllm_text::tokenizer::Tokenizer;

197 + use vllm_tokenizer::test_utils::TestTokenizer;

198

199 use super::prepare_completion_request;

200 use crate::lora::LoraModelResolution;

@@ -212,32 +212,8 @@ mod tests {

212 }

213 }

214

215 - #[derive(Debug)]

216 - struct TestTokenizer;

217 -

218 - impl Tokenizer for TestTokenizer {

219 - fn encode(

220 - &self,

221 - text: &str,

222 - _add_special_tokens: bool,

223 - ) -> vllm_text::tokenizer::Result<Vec<u32>> {

224 - Ok(text.bytes().map(u32::from).collect())

225 - }

226 -

227 - fn decode(

228 - &self,

229 - token_ids: &[u32],

230 - _skip_special_tokens: bool,

231 - ) -> vllm_text::tokenizer::Result<String> {

232 - Ok(

233 - String::from_utf8_lossy(&token_ids.iter().map(|id| *id as u8).collect::<Vec<_>>())

234 - .into_owned(),

235 - )

236 - }

237 -

238 - fn token_to_id(&self, _token: &str) -> Option<u32> {

239 - None

240 - }

215 + fn test_tokenizer() -> TestTokenizer {

216 + TestTokenizer::new()

217 }

218

219 fn base_request_json() -> serde_json::Value {

@@ -297,7 +273,7 @@ mod tests {

273 request,

274 &served(&["Qwen/Qwen1.5-0.5B-Chat"]),

275 ResolvedRequestContext::default(),

300 - &TestTokenizer,

276 + &test_tokenizer(),

277 )

278 .expect("prepare");

279

@@ -340,7 +316,7 @@ mod tests {

316 request,

317 &served(&["Qwen/Qwen1.5-0.5B-Chat"]),

318 ResolvedRequestContext::default(),

343 - &TestTokenizer,

319 + &test_tokenizer(),

320 )

321 .expect("prepare")

322 .text_request

@@ -374,7 +350,7 @@ mod tests {

350 request,

351 &served(&["Qwen/Qwen1.5-0.5B-Chat"]),

352 ResolvedRequestContext::default(),

377 - &TestTokenizer,

353 + &test_tokenizer(),

354 )

355 .expect("prepare");

356

@@ -399,7 +375,7 @@ mod tests {

375 request,

376 &served(&["Qwen/Qwen1.5-0.5B-Chat"]),

377 ResolvedRequestContext::default(),

402 - &TestTokenizer,

378 + &test_tokenizer(),

379 )

380 .expect("prepare");

381

@@ -422,7 +398,7 @@ mod tests {

398 request,

399 &served(&["Qwen/Qwen1.5-0.5B-Chat"]),

400 ResolvedRequestContext::default(),

425 - &TestTokenizer,

401 + &test_tokenizer(),

402 )

403 .expect("prepare");

404

@@ -446,7 +422,7 @@ mod tests {

422 request,

423 &served(&["Qwen/Qwen1.5-0.5B-Chat"]),

424 ResolvedRequestContext::default(),

449 - &TestTokenizer,

425 + &test_tokenizer(),

426 )

427 .expect("prepare");

428

@@ -471,7 +447,7 @@ mod tests {

447 request,

448 &served(&["Qwen/Qwen1.5-0.5B-Chat"]),

449 ResolvedRequestContext::default(),

474 - &TestTokenizer,

450 + &test_tokenizer(),

451 )

452 .expect("prepare");

453

@@ -498,7 +474,7 @@ mod tests {

474 request,

475 &served(&["Qwen/Qwen1.5-0.5B-Chat"]),

476 ResolvedRequestContext::default(),

501 - &TestTokenizer,

477 + &test_tokenizer(),

478 )

479 .expect("prepare");

480

@@ -523,7 +499,7 @@ mod tests {

499 request,

500 &served(&["Qwen/Qwen1.5-0.5B-Chat"]),

501 ResolvedRequestContext::default(),

526 - &TestTokenizer,

502 + &test_tokenizer(),

503 )

504 .expect("prepare");

505

@@ -549,7 +525,7 @@ mod tests {

525 request,

526 &served(&["Qwen/Qwen1.5-0.5B-Chat"]),

527 ResolvedRequestContext::default(),

552 - &TestTokenizer,

528 + &test_tokenizer(),

529 )

530 .expect("prepare");

531 assert_eq!(prepared.text_request.sampling_params.logprobs, Some(1));

@@ -574,7 +550,7 @@ mod tests {

550 request,

551 &served(&["Qwen/Qwen1.5-0.5B-Chat"]),

552 request_context(&headers, None),

577 - &TestTokenizer,

553 + &test_tokenizer(),

554 )

555 .expect("prepare");

556 assert_eq!(prepared.text_request.data_parallel_rank, Some(3));

@@ -593,7 +569,7 @@ mod tests {

569 request,

570 &served(&["Qwen/Qwen1.5-0.5B-Chat"]),

571 ResolvedRequestContext::default(),

596 - &TestTokenizer,

572 + &test_tokenizer(),

573 )

574 .expect("prepare");

575 assert_eq!(prepared.text_request.data_parallel_rank, None);

rust/src/server/src/routes/tests.rs

+23 -146

@@ -40,8 +40,9 @@ use vllm_engine_core_client::{

40 };

41 use vllm_llm::Llm;

42 use vllm_metrics::METRICS;

43 -use vllm_text::tokenizer::{DynTokenizer, Tokenizer};

43 +use vllm_text::tokenizer::DynTokenizer;

44 use vllm_text::{Prompt, TextBackend};

45 +use vllm_tokenizer::test_utils::TestTokenizer;

46 use zeromq::prelude::{SocketRecv, SocketSend};

47 use zeromq::{DealerSocket, PushSocket, ZmqMessage};

@@ -417,82 +418,20 @@ struct FakeChatBackend {

418 }

419

420 /// Synthetic BOS id used when `add_special_tokens` is true in tests.

420 -const FAKE_BOS_TOKEN_ID: u32 = 1;

421 -

422 -#[derive(Debug)]

423 -struct FakeChatTokenizer;

424 -

425 -impl Tokenizer for FakeChatTokenizer {

426 - fn encode(

427 - &self,

428 - text: &str,

429 - add_special_tokens: bool,

430 - ) -> vllm_text::tokenizer::Result<Vec<u32>> {

431 - let mut token_ids = Vec::new();

432 - if add_special_tokens {

433 - token_ids.push(FAKE_BOS_TOKEN_ID);

434 - }

435 - let mut rest = text;

436 - while !rest.is_empty() {

437 - if let Some(stripped) = rest.strip_prefix("<image>") {

438 - token_ids.push(999);

439 - rest = stripped;

440 - continue;

441 - }

442 - if let Some(stripped) = rest.strip_prefix("<|image_pad|>") {

443 - token_ids.push(151655);

444 - rest = stripped;

445 - continue;

446 - }

447 -

448 - let ch = rest.chars().next().expect("rest is not empty");

449 - let mut buf = [0; 4];

450 - token_ids.extend(ch.encode_utf8(&mut buf).bytes().map(u32::from));

451 - rest = &rest[ch.len_utf8()..];

452 - }

453 - Ok(token_ids)

454 - }

455 -

456 - fn decode(

457 - &self,

458 - token_ids: &[u32],

459 - _skip_special_tokens: bool,

460 - ) -> vllm_text::tokenizer::Result<String> {

461 - Ok(

462 - String::from_utf8_lossy(&token_ids.iter().map(|id| *id as u8).collect::<Vec<_>>())

463 - .into_owned(),

464 - )

465 - }

466 -

467 - fn token_to_id(&self, token: &str) -> Option<u32> {

468 - match token {

469 - "<image>" => Some(999),

470 - "<|image_pad|>" => Some(151655),

471 - "<think>" => Some(0xF001),

472 - "</think>" => Some(0xF002),

473 - "<|START_THINKING|>" => Some(0xF003),

474 - "<|END_THINKING|>" => Some(0xF004),

475 - "◁think▷" => Some(0xF005),

476 - "◁/think▷" => Some(0xF006),

477 - _ => None,

478 - }

479 - }

480 -

481 - fn id_to_token(&self, id: u32) -> Option<String> {

482 - match id {

483 - FAKE_BOS_TOKEN_ID => Some("<bos>".to_string()),

484 - 999 => Some("<image>".to_string()),

485 - 151655 => Some("<|image_pad|>".to_string()),

486 - 0xF001 => Some("<think>".to_string()),

487 - 0xF002 => Some("</think>".to_string()),

488 - 0xF003 => Some("<|START_THINKING|>".to_string()),

489 - 0xF004 => Some("<|END_THINKING|>".to_string()),

490 - 0xF005 => Some("◁think▷".to_string()),

491 - 0xF006 => Some("◁/think▷".to_string()),

492 - id if id < 128 => char::from_u32(id).map(|ch| ch.to_string()),

493 - _ => None,

494 - }

495 - }

421 +const FAKE_BOS_TOKEN_ID: u32 = 256;

422 +const UNKNOWN_DECODE_TOKEN_ID: u32 = 10_000;

423 +

424 +fn fake_chat_tokenizer() -> TestTokenizer {

425 + TestTokenizer::new()

426 + .with_bos_token("<bos>", FAKE_BOS_TOKEN_ID)

427 + .with_regular_token("<image>", 999)

428 + .with_regular_token("<|image_pad|>", 151655)

429 + .with_regular_token("<think>", 0xF001)

430 + .with_regular_token("</think>", 0xF002)

431 + .with_regular_token("<|START_THINKING|>", 0xF003)

432 + .with_regular_token("<|END_THINKING|>", 0xF004)

433 + .with_regular_token("◁think▷", 0xF005)

434 + .with_regular_token("◁/think▷", 0xF006)

435 }

436

437 impl FakeChatBackend {

@@ -530,7 +469,7 @@ impl fmt::Debug for FakeChatBackend {

469

470 impl TextBackend for FakeChatBackend {

471 fn tokenizer(&self) -> DynTokenizer {

533 - Arc::new(FakeChatTokenizer)

472 + Arc::new(fake_chat_tokenizer())

473 }

474

475 fn model_id(&self) -> &str {

@@ -630,7 +569,7 @@ fn qwen_multimodal_model_info() -> vllm_chat::multimodal::MultimodalModelInfo {

569 Some("qwen2_vl".to_string()),

570 Some(&config_path),

571 None,

633 - Arc::new(FakeChatTokenizer),

572 + Arc::new(fake_chat_tokenizer()),

573 )

574 .expect("load multimodal info")

575 .expect("qwen multimodal info is registered");

@@ -638,70 +577,6 @@ fn qwen_multimodal_model_info() -> vllm_chat::multimodal::MultimodalModelInfo {

577 info

578 }

579

641 -#[derive(Clone, Debug)]

642 -struct FailingDecodeChatBackend;

643 -

644 -#[derive(Debug)]

645 -struct FailingDecodeTokenizer;

646 -

647 -impl Tokenizer for FailingDecodeTokenizer {

648 - fn encode(

649 - &self,

650 - text: &str,

651 - add_special_tokens: bool,

652 - ) -> vllm_text::tokenizer::Result<Vec<u32>> {

653 - FakeChatTokenizer.encode(text, add_special_tokens)

654 - }

655 -

656 - fn decode(

657 - &self,

658 - token_ids: &[u32],

659 - skip_special_tokens: bool,

660 - ) -> vllm_text::tokenizer::Result<String> {

661 - if token_ids.contains(&(b'i' as u32)) {

662 - return Err(vllm_text::tokenizer::TokenizerError(

663 - "forced decode failure for streaming test".to_string(),

664 - ));

665 - }

666 -

667 - FakeChatTokenizer.decode(token_ids, skip_special_tokens)

668 - }

669 -

670 - fn token_to_id(&self, token: &str) -> Option<u32> {

671 - FakeChatTokenizer.token_to_id(token)

672 - }

673 -}

674 -

675 -impl TextBackend for FailingDecodeChatBackend {

676 - fn tokenizer(&self) -> DynTokenizer {

677 - Arc::new(FailingDecodeTokenizer)

678 - }

679 -

680 - fn model_id(&self) -> &str {

681 - "test-model"

682 - }

683 -}

684 -

685 -impl ChatBackend for FailingDecodeChatBackend {

686 - fn chat_renderer(&self) -> DynChatRenderer {

687 - Arc::new(self.clone())

688 - }

689 -

690 - fn new_chat_output_processor(

691 - &self,

692 - _request: &mut ChatRequest,

693 - _options: NewChatOutputProcessorOptions<'_>,

694 - ) -> vllm_chat::Result<DynChatOutputProcessor> {

695 - Ok(Box::new(DefaultChatOutputProcessor::plain_text_only()))

696 - }

697 -}

698 -

699 -impl ChatRenderer for FailingDecodeChatBackend {

700 - fn render(&self, request: &ChatRequest) -> vllm_chat::Result<vllm_chat::RenderedPrompt> {

701 - FakeChatBackend::new().render(request)

702 - }

703 -}

704 -

580 async fn test_models_with_engine_outputs_and_backend_inner(

581 engine_id: impl Into<EngineId>,

582 output_specs: Vec<(Vec<u32>, Option<EngineCoreFinishReason>)>,

@@ -2755,8 +2630,8 @@ async fn load_endpoint_resets_when_stream_response_is_dropped() {

2630 #[serial]

2631 async fn stream_error_is_returned_as_openai_error_sse() {

2632 let (app, engine_task) = test_app_with_backend_and_stream_output_specs(

2758 - Arc::new(FailingDecodeChatBackend),

2759 - default_stream_output_specs(),

2633 + Arc::new(FakeChatBackend::new()),

2634 + vec![(vec![UNKNOWN_DECODE_TOKEN_ID], None)],

2635 )

2636 .await;

2637 let response = app

@@ -2789,7 +2664,9 @@ async fn stream_error_is_returned_as_openai_error_sse() {

2664 assert!(text.contains("\"role\":\"assistant\""), "{text}");

2665 assert!(text.contains("\"type\":\"server_error\""), "{text}");

2666 assert!(

2792 - text.contains("forced decode failure for streaming test"),

2667 + text.contains(&format!(

2668 + "test tokenizer cannot decode unknown token id {UNKNOWN_DECODE_TOKEN_ID}"

2669 + )),

2670 "{text}"

2671 );

2672 assert!(!text.contains("\"usage\":"), "{text}");

rust/src/text/Cargo.toml

@@ -31,6 +31,7 @@ serial_test.workspace = true

31 tempfile.workspace = true

32 tokio.workspace = true

33 vllm-llm = { workspace = true, features = ["test-util"] }

34 +vllm-tokenizer = { workspace = true, features = ["test-utils"] }

36 [lints]

37 workspace = true

rust/src/text/src/lower.rs

+4 -57

@@ -275,6 +275,7 @@ mod tests {

275 use std::collections::{BTreeSet, HashMap};

276

277 use serial_test::file_serial;

278 + use vllm_tokenizer::test_utils::TestTokenizer;

279

280 use super::*;

281 use crate::backend::hf::HfTextBackend;

@@ -282,60 +283,8 @@ mod tests {

283 use crate::error::{LogprobsError, TokenIdsError};

284 use crate::request::{Prompt, TextRequest};

285

285 - /// Stub tokenizer that returns empty token IDs — sufficient for tests that

286 - /// don't exercise bad-words tokenization.

287 - struct StubTokenizer;

288 -

289 - impl Tokenizer for StubTokenizer {

290 - fn encode(

291 - &self,

292 - _text: &str,

293 - _add_special_tokens: bool,

294 - ) -> vllm_tokenizer::Result<Vec<u32>> {

295 - Ok(vec![])

296 - }

297 -

298 - fn decode(

299 - &self,

300 - _token_ids: &[u32],

301 - _skip_special_tokens: bool,

302 - ) -> vllm_tokenizer::Result<String> {

303 - Ok(String::new())

304 - }

305 -

306 - fn token_to_id(&self, _token: &str) -> Option<u32> {

307 - None

308 - }

309 - }

310 -

311 - fn stub_tokenizer() -> StubTokenizer {

312 - StubTokenizer

313 - }

314 -

315 - struct FixedTokenizer {

316 - token_ids: Vec<u32>,

317 - }

318 -

319 - impl Tokenizer for FixedTokenizer {

320 - fn encode(

321 - &self,

322 - _text: &str,

323 - _add_special_tokens: bool,

324 - ) -> vllm_tokenizer::Result<Vec<u32>> {

325 - Ok(self.token_ids.clone())

326 - }

327 -

328 - fn decode(

329 - &self,

330 - _token_ids: &[u32],

331 - _skip_special_tokens: bool,

332 - ) -> vllm_tokenizer::Result<String> {

333 - Ok(String::new())

334 - }

335 -

336 - fn token_to_id(&self, _token: &str) -> Option<u32> {

337 - None

338 - }

286 + fn stub_tokenizer() -> TestTokenizer {

287 + TestTokenizer::new()

288 }

289

290 fn sample_request() -> TextRequest {

@@ -952,9 +901,7 @@ mod tests {

901

902 #[test]

903 fn lower_sampling_params_rejects_out_of_vocab_bad_words() {

955 - let tokenizer = FixedTokenizer {

956 - token_ids: vec![1999, 2000],

957 - };

904 + let tokenizer = TestTokenizer::new().with_regular_token("blocked", 2000);

905 let error = lower_sampling_params(

906 SamplingParams {

907 bad_words: Some(vec!["blocked".to_string()]),

rust/src/text/src/output/decoded.rs

+3 -29

@@ -323,37 +323,11 @@ mod tests {

323 use futures::{Stream, stream};

324 use vllm_engine_core_client::AbortCause;

325 use vllm_llm::GenerateOutput;

326 - use vllm_tokenizer::Tokenizer;

326 + use vllm_tokenizer::test_utils::TestTokenizer;

327

328 use super::*;

329 use crate::output::TextOutputStreamExt as _;

330

331 - /// Backend that treats each token ID as a raw byte, producing lossy UTF-8.

332 - struct ByteTokenizer;

333 -

334 - impl Tokenizer for ByteTokenizer {

335 - fn encode(

336 - &self,

337 - _text: &str,

338 - _add_special_tokens: bool,

339 - ) -> vllm_tokenizer::Result<Vec<u32>> {

340 - unreachable!()

341 - }

342 -

343 - fn decode(

344 - &self,

345 - token_ids: &[u32],

346 - _skip_special_tokens: bool,

347 - ) -> vllm_tokenizer::Result<String> {

348 - let bytes = token_ids.iter().map(|id| *id as u8).collect::<Vec<_>>();

349 - Ok(String::from_utf8_lossy(&bytes).into_owned())

350 - }

351 -

352 - fn token_to_id(&self, _token: &str) -> Option<u32> {

353 - unreachable!()

354 - }

355 - }

356 -

331 /// Helper: run `decoded_text_event_stream` to completion and return the

332 /// collected output.

333 async fn run_to_completion(

@@ -366,7 +340,7 @@ mod tests {

340 token_ids,

341 Some(FinishReason::Length),

342 ))]);

369 - let tokenizer: DynTokenizer = Arc::new(ByteTokenizer);

343 + let tokenizer: DynTokenizer = Arc::new(TestTokenizer::new());

344 decoded_text_event_stream("test".into(), tokenizer, raw_stream, decode_options, false)

345 .collect_output()

346 .await

@@ -419,7 +393,7 @@ mod tests {

393 ))),

394 dropped_cause: Arc::clone(&dropped_cause),

395 };

422 - let tokenizer: DynTokenizer = Arc::new(ByteTokenizer);

396 + let tokenizer: DynTokenizer = Arc::new(TestTokenizer::new());

397

398 let output = decoded_text_event_stream(

399 "test".into(),

rust/src/text/src/output/logprobs.rs

+3 -30

@@ -129,40 +129,13 @@ fn decode_position_logprobs<T: Tokenizer + ?Sized>(

129 #[cfg(test)]

130 mod tests {

131 use vllm_llm::{Logprobs, PositionLogprobs, TokenLogprob};

132 + use vllm_tokenizer::test_utils::TestTokenizer;

133

134 use super::*;

135

135 - #[derive(Debug)]

136 - struct ByteTokenizer;

137 -

138 - impl vllm_tokenizer::Tokenizer for ByteTokenizer {

139 - fn encode(

140 - &self,

141 - _text: &str,

142 - _add_special_tokens: bool,

143 - ) -> vllm_tokenizer::Result<Vec<u32>> {

144 - unreachable!()

145 - }

146 -

147 - fn decode(

148 - &self,

149 - token_ids: &[u32],

150 - _skip_special_tokens: bool,

151 - ) -> vllm_tokenizer::Result<String> {

152 - Ok(String::from_utf8_lossy(

153 - &token_ids.iter().map(|token_id| *token_id as u8).collect::<Vec<_>>(),

154 - )

155 - .into_owned())

156 - }

157 -

158 - fn token_to_id(&self, _token: &str) -> Option<u32> {

159 - unreachable!()

160 - }

161 - }

162 -

136 #[test]

137 fn decode_logprobs_decodes_every_candidate_token() {

165 - let tokenizer = ByteTokenizer;

138 + let tokenizer = TestTokenizer::new();

139 let logprobs = Logprobs {

140 positions: vec![PositionLogprobs {

141 entries: vec![

@@ -205,7 +178,7 @@ mod tests {

178

179 #[test]

180 fn decode_prompt_logprobs_separates_first_prompt_token() {

208 - let tokenizer = ByteTokenizer;

181 + let tokenizer = TestTokenizer::new();

182 let logprobs = Logprobs {

183 positions: vec![PositionLogprobs {

184 entries: vec![TokenLogprob {

rust/src/tokenizer/Cargo.toml

@@ -4,6 +4,9 @@ version.workspace = true

4 edition.workspace = true

5 license.workspace = true

7 +[features]

8 +test-utils = []

9 +

10 [dependencies]

11 base64.workspace = true

12 fastokens.workspace = true

rust/src/tokenizer/src/incremental.rs

+12

@@ -170,6 +170,10 @@ mod tests {

170 fn token_to_id(&self, _token: &str) -> Option<u32> {

171 unreachable!()

172 }

173 +

174 + fn id_to_token(&self, _id: u32) -> Option<String> {

175 + unreachable!()

176 + }

177 }

178

179 #[test]

@@ -248,6 +252,10 @@ mod tests {

252 fn token_to_id(&self, _token: &str) -> Option<u32> {

253 unreachable!()

254 }

255 +

256 + fn id_to_token(&self, _id: u32) -> Option<String> {

257 + unreachable!()

258 + }

259 }

260

261 #[test]

@@ -320,6 +328,10 @@ mod tests {

328 fn token_to_id(&self, _token: &str) -> Option<u32> {

329 unreachable!()

330 }

331 +

332 + fn id_to_token(&self, _id: u32) -> Option<String> {

333 + unreachable!()

334 + }

335 }

336

337 /// Without the char-boundary fix, this panics slicing mid-emoji.

rust/src/tokenizer/src/lib.rs

+3 -5

@@ -8,6 +8,8 @@ mod error;

8 mod hf;

9 mod incremental;

10 mod tekken;

11 +#[cfg(any(test, feature = "test-utils"))]

12 +pub mod test_utils;

13 mod tiktoken;

15 pub use error::{Result, TokenizerError};

@@ -28,11 +30,7 @@ pub trait Tokenizer: Send + Sync {

30 fn token_to_id(&self, token: &str) -> Option<u32>;

32 /// Convert one token ID into the tokenizer's raw token string.

31 - fn id_to_token(&self, _id: u32) -> Option<String> {

32 - // TODO: remove default impl and require this to be implemented by all

33 - // tokenizers

34 - None

35 - }

33 + fn id_to_token(&self, id: u32) -> Option<String>;

35 /// Return the vocabulary size. Backends that cannot report it fall back to

36 /// `usize::MAX`, an effectively unbounded value used only by test stubs.

rust/src/tokenizer/src/test_utils.rs new

+434

@@ -0,0 +1,434 @@

1 +use std::collections::BTreeMap;

2 +

3 +use crate::{Result, Tokenizer, TokenizerError};

4 +

5 +const FIRST_CONFIGURED_TOKEN_ID: u32 = 256;

6 +

7 +/// Whether a configured test token should be treated as special.

8 +///

9 +/// Special tokens are skipped by [`Tokenizer::decode`] when

10 +/// `skip_special_tokens` is set. Regular configured tokens are always emitted.

11 +#[derive(Debug, Clone, Copy, PartialEq, Eq)]

12 +pub enum TestTokenKind {

13 + /// Token is skipped when `skip_special_tokens = true`.

14 + Special,

15 + /// Token is emitted regardless of `skip_special_tokens`.

16 + Regular,

17 +}

18 +

19 +impl TestTokenKind {

20 + fn is_special(self) -> bool {

21 + matches!(self, Self::Special)

22 + }

23 +}

24 +

25 +/// Decode behavior for token ids that are neither configured tokens nor byte ids.

26 +///

27 +/// The default is [`UnknownDecode::Error`] so tests notice missing tokenizer

28 +/// fixtures instead of silently accepting impossible ids. Individual tests can

29 +/// opt into empty or replacement output when they are explicitly modeling a

30 +/// lenient detokenization path.

31 +#[derive(Debug, Clone, Copy, PartialEq, Eq)]

32 +pub enum UnknownDecode {

33 + /// Return a tokenizer error on the first unknown id.

34 + Error,

35 + /// Drop unknown ids from decoded output.

36 + Empty,

37 + /// Emit U+FFFD for each unknown id.

38 + Replacement,

39 +}

40 +

41 +#[derive(Debug, Clone)]

42 +struct TestToken {

43 + text: String,

44 + kind: TestTokenKind,

45 +}

46 +

47 +/// Configurable tokenizer for Rust frontend tests.

48 +///

49 +/// `TestTokenizer` is intentionally small, but its methods obey the same basic

50 +/// contract as production tokenizers:

51 +///

52 +/// - ordinary text encodes as UTF-8 byte ids;

53 +/// - configured token ids start at 256, leaving `0..=255` for byte fallback;

54 +/// - configured token ids and token text are unique;

55 +/// - configured tokens are matched before ordinary bytes, using longest-prefix matching so

56 +/// multi-character markers such as `<think>` work naturally;

57 +/// - `token_to_id` and `id_to_token` are consistent for configured tokens;

58 +/// - `decode` is strict by default for ids outside the byte range and the configured token table;

59 +/// - `vocab_size` is an exclusive upper bound covering byte ids and configured token ids unless a

60 +/// test sets it explicitly.

61 +///

62 +/// Prefer this helper over ad-hoc fake tokenizers for tests that rely on

63 +/// tokenizer semantics. Keep dedicated tiny fakes for error injection or for

64 +/// tests that deliberately need a degenerate tokenizer.

65 +#[derive(Debug, Clone)]

66 +pub struct TestTokenizer {

67 + token_to_id: BTreeMap<String, u32>,

68 + id_to_token: BTreeMap<u32, TestToken>,

69 + unknown_decode: UnknownDecode,

70 + vocab_size: Option<usize>,

71 + bos_token_id: Option<u32>,

72 +}

73 +

74 +impl Default for TestTokenizer {

75 + fn default() -> Self {

76 + Self::new()

77 + }

78 +}

79 +

80 +impl TestTokenizer {

81 + /// Create a byte-level test tokenizer with strict unknown-id decode.

82 + pub fn new() -> Self {

83 + Self {

84 + token_to_id: BTreeMap::new(),

85 + id_to_token: BTreeMap::new(),

86 + unknown_decode: UnknownDecode::Error,

87 + vocab_size: None,

88 + bos_token_id: None,

89 + }

90 + }

91 +

92 + /// Add a configured token and return the updated tokenizer.

93 + ///

94 + /// Configured tokens must use ids outside the byte range and may be marked

95 + /// special or regular.

96 + pub fn with_token(mut self, token: impl Into<String>, id: u32, kind: TestTokenKind) -> Self {

97 + self.insert_token(token, id, kind);

98 + self

99 + }

100 +

101 + /// Add a special configured token and return the updated tokenizer.

102 + pub fn with_special_token(self, token: impl Into<String>, id: u32) -> Self {

103 + self.with_token(token, id, TestTokenKind::Special)

104 + }

105 +

106 + /// Add a regular configured token and return the updated tokenizer.

107 + pub fn with_regular_token(self, token: impl Into<String>, id: u32) -> Self {

108 + self.with_token(token, id, TestTokenKind::Regular)

109 + }

110 +

111 + /// Add a special BOS token inserted by `encode(..., true)`.

112 + ///

113 + /// This also registers the token in the normal token/id maps so

114 + /// `token_to_id`, `id_to_token`, `decode`, and `is_special_id` stay

115 + /// consistent for the inserted id.

116 + pub fn with_bos_token(mut self, token: impl Into<String>, id: u32) -> Self {

117 + self.insert_token(token, id, TestTokenKind::Special);

118 + self.bos_token_id = Some(id);

119 + self

120 + }

121 +

122 + /// Set decode behavior for unknown non-byte ids.

123 + pub fn with_unknown_decode(mut self, behavior: UnknownDecode) -> Self {

124 + self.unknown_decode = behavior;

125 + self

126 + }

127 +

128 + /// Set an explicit vocabulary size.

129 + ///

130 + /// Use this when a test needs a model-like vocabulary bound that differs

131 + /// from the highest configured token id plus one.

132 + pub fn with_vocab_size(mut self, vocab_size: usize) -> Self {

133 + self.vocab_size = Some(vocab_size);

134 + self

135 + }

136 +

137 + fn insert_token(&mut self, token: impl Into<String>, id: u32, kind: TestTokenKind) {

138 + let token = token.into();

139 + assert!(

140 + !token.is_empty(),

141 + "configured test token text must be non-empty"

142 + );

143 + assert!(

144 + id >= FIRST_CONFIGURED_TOKEN_ID,

145 + "configured test token id {id} overlaps byte fallback range 0..=255"

146 + );

147 + assert!(

148 + token.len() > 1,

149 + "configured test token text {token:?} overlaps byte fallback token text"

150 + );

151 + if self.token_to_id.insert(token.clone(), id).is_some() {

152 + panic!("configured test token text {token:?} was registered more than once");

153 + }

154 + if self.id_to_token.insert(id, TestToken { text: token, kind }).is_some() {

155 + panic!("configured test token id {id} was registered more than once");

156 + }

157 + }

158 +

159 + fn byte_to_token(id: u32) -> Option<String> {

160 + u8::try_from(id).ok().map(|byte| String::from_utf8_lossy(&[byte]).into_owned())

161 + }

162 +

163 + fn flush_bytes(bytes: &mut Vec<u8>, output: &mut String) {

164 + if !bytes.is_empty() {

165 + output.push_str(&String::from_utf8_lossy(bytes));

166 + bytes.clear();

167 + }

168 + }

169 +

170 + fn configured_token_prefix(&self, text: &str) -> Option<(&str, u32)> {

171 + self.token_to_id

172 + .iter()

173 + .filter_map(|(token, &id)| text.starts_with(token).then_some((token.as_str(), id)))

174 + .max_by_key(|(token, _)| token.len())

175 + }

176 +

177 + fn inferred_vocab_size(&self) -> usize {

178 + let max_configured =

179 + self.id_to_token.last_key_value().map(|(&id, _)| id as usize + 1).unwrap_or(0);

180 + 256.max(max_configured)

181 + }

182 +}

183 +

184 +impl Tokenizer for TestTokenizer {

185 + fn encode(&self, text: &str, add_special_tokens: bool) -> Result<Vec<u32>> {

186 + let mut ids = Vec::new();

187 + if add_special_tokens && let Some(bos_token_id) = self.bos_token_id {

188 + ids.push(bos_token_id);

189 + }

190 +

191 + let mut rest = text;

192 + while !rest.is_empty() {

193 + if let Some((token, id)) = self.configured_token_prefix(rest) {

194 + ids.push(id);

195 + rest = &rest[token.len()..];

196 + continue;

197 + }

198 +

199 + let ch = rest.chars().next().expect("rest is not empty");

200 + let mut buf = [0_u8; 4];

201 + ids.extend(ch.encode_utf8(&mut buf).bytes().map(u32::from));

202 + rest = &rest[ch.len_utf8()..];

203 + }

204 +

205 + Ok(ids)

206 + }

207 +

208 + fn decode(&self, token_ids: &[u32], skip_special_tokens: bool) -> Result<String> {

209 + let mut output = String::new();

210 + let mut pending_bytes = Vec::new();

211 + for &id in token_ids {

212 + if let Some(token) = self.id_to_token.get(&id) {

213 + Self::flush_bytes(&mut pending_bytes, &mut output);

214 + if !(skip_special_tokens && token.kind.is_special()) {

215 + output.push_str(&token.text);

216 + }

217 + } else if let Ok(byte) = u8::try_from(id) {

218 + pending_bytes.push(byte);

219 + } else {

220 + Self::flush_bytes(&mut pending_bytes, &mut output);

221 + match self.unknown_decode {

222 + UnknownDecode::Error => {

223 + return Err(TokenizerError(format!(

224 + "test tokenizer cannot decode unknown token id {id}"

225 + )));

226 + }

227 + UnknownDecode::Empty => {}

228 + UnknownDecode::Replacement => output.push('\u{FFFD}'),

229 + }

230 + }

231 + }

232 + Self::flush_bytes(&mut pending_bytes, &mut output);

233 + Ok(output)

234 + }

235 +

236 + fn token_to_id(&self, token: &str) -> Option<u32> {

237 + self.token_to_id.get(token).copied().or_else(|| {

238 + let bytes = token.as_bytes();

239 + (bytes.len() == 1).then(|| u32::from(bytes[0]))

240 + })

241 + }

242 +

243 + fn id_to_token(&self, id: u32) -> Option<String> {

244 + self.id_to_token

245 + .get(&id)

246 + .map(|token| token.text.clone())

247 + .or_else(|| Self::byte_to_token(id))

248 + }

249 +

250 + fn vocab_size(&self) -> usize {

251 + self.vocab_size.unwrap_or_else(|| self.inferred_vocab_size())

252 + }

253 +

254 + fn is_special_id(&self, token_id: u32) -> bool {

255 + self.id_to_token.get(&token_id).is_some_and(|token| token.kind.is_special())

256 + }

257 +}

258 +

259 +#[cfg(test)]

260 +mod tests {

261 + use super::*;

262 +

263 + #[test]

264 + fn byte_text_roundtrips_and_reports_byte_ids() {

265 + let tokenizer = TestTokenizer::new();

266 +

267 + let ids = tokenizer.encode("hi", false).unwrap();

268 + assert_eq!(ids, vec![b'h' as u32, b'i' as u32]);

269 + assert_eq!(tokenizer.decode(&ids, false).unwrap(), "hi");

270 + assert_eq!(tokenizer.token_to_id("h"), Some(b'h' as u32));

271 + assert_eq!(tokenizer.id_to_token(b'h' as u32).as_deref(), Some("h"));

272 + assert_eq!(tokenizer.vocab_size(), 256);

273 + }

274 +

275 + #[test]

276 + fn configured_tokens_use_longest_prefix_matching() {

277 + let tokenizer = TestTokenizer::new()

278 + .with_regular_token("<image>", 999)

279 + .with_regular_token("<image></image>", 1000);

280 +

281 + assert_eq!(

282 + tokenizer.encode("a<image></image>b", false).unwrap(),

283 + vec![b'a' as u32, 1000, b'b' as u32,]

284 + );

285 + assert_eq!(

286 + tokenizer.decode(&[b'a' as u32, 1000, b'b' as u32], false).unwrap(),

287 + "a<image></image>b"

288 + );

289 + assert_eq!(tokenizer.token_to_id("<image>"), Some(999));

290 + assert_eq!(

291 + tokenizer.id_to_token(1000).as_deref(),

292 + Some("<image></image>")

293 + );

294 + assert_eq!(tokenizer.vocab_size(), 1001);

295 + }

296 +

297 + #[test]

298 + fn non_ascii_text_roundtrips_through_buffered_byte_decode() {

299 + let tokenizer = TestTokenizer::new();

300 + let text = "你好, café, 🚀";

301 +

302 + let ids = tokenizer.encode(text, false).unwrap();

303 + assert_eq!(

304 + ids,

305 + text.as_bytes().iter().copied().map(u32::from).collect::<Vec<_>>()

306 + );

307 + assert_eq!(tokenizer.decode(&ids, false).unwrap(), text);

308 + }

309 +

310 + #[test]

311 + fn buffered_byte_decode_flushes_around_configured_tokens() {

312 + let tokenizer = TestTokenizer::new()

313 + .with_regular_token("<image>", 999)

314 + .with_special_token("<skip>", 1000);

315 +

316 + assert_eq!(

317 + tokenizer.encode("你<image>好<skip>🚀", false).unwrap(),

318 + vec![228, 189, 160, 999, 229, 165, 189, 1000, 240, 159, 154, 128]

319 + );

320 + assert_eq!(

321 + tokenizer

322 + .decode(

323 + &[228, 189, 160, 999, 229, 165, 189, 1000, 240, 159, 154, 128],

324 + false

325 + )

326 + .unwrap(),

327 + "你<image>好<skip>🚀"

328 + );

329 + assert_eq!(

330 + tokenizer

331 + .decode(

332 + &[228, 189, 160, 999, 229, 165, 189, 1000, 240, 159, 154, 128],

333 + true

334 + )

335 + .unwrap(),

336 + "你<image>好🚀"

337 + );

338 + }

339 +

340 + #[test]

341 + fn invalid_utf8_bytes_decode_lossily_as_a_sequence() {

342 + let tokenizer = TestTokenizer::new();

343 +

344 + assert_eq!(tokenizer.decode(&[0xE4, 0xBD], false).unwrap(), "\u{FFFD}");

345 + assert_eq!(

346 + tokenizer.decode(&[0xFF, b'a' as u32], false).unwrap(),

347 + "\u{FFFD}a"

348 + );

349 + }

350 +

351 + #[test]

352 + fn special_tokens_respect_skip_special_tokens() {

353 + let tokenizer = TestTokenizer::new()

354 + .with_bos_token("<bos>", 256)

355 + .with_special_token("<think>", 0xF001)

356 + .with_regular_token("</think>", 0xF002);

357 +

358 + assert_eq!(

359 + tokenizer.encode("<think>x</think>", true).unwrap(),

360 + vec![256, 0xF001, b'x' as u32, 0xF002,]

361 + );

362 + assert_eq!(

363 + tokenizer.decode(&[256, 0xF001, b'x' as u32, 0xF002], false).unwrap(),

364 + "<bos><think>x</think>"

365 + );

366 + assert_eq!(

367 + tokenizer.decode(&[256, 0xF001, b'x' as u32, 0xF002], true).unwrap(),

368 + "x</think>"

369 + );

370 + assert!(tokenizer.is_special_id(0xF001));

371 + assert!(!tokenizer.is_special_id(0xF002));

372 + }

373 +

374 + #[test]

375 + #[should_panic(expected = "configured test token id 255 overlaps byte fallback range 0..=255")]

376 + fn configured_token_id_must_stay_outside_byte_range() {

377 + let _ = TestTokenizer::new().with_regular_token("<token>", 255);

378 + }

379 +

380 + #[test]

381 + #[should_panic(expected = "configured test token text \"a\" overlaps byte fallback token text")]

382 + fn configured_token_text_must_not_shadow_byte_tokens() {

383 + let _ = TestTokenizer::new().with_regular_token("a", 256);

384 + }

385 +

386 + #[test]

387 + #[should_panic(

388 + expected = "configured test token text \"<token>\" was registered more than once"

389 + )]

390 + fn configured_token_text_must_be_unique() {

391 + let _ = TestTokenizer::new()

392 + .with_regular_token("<token>", 256)

393 + .with_regular_token("<token>", 257);

394 + }

395 +

396 + #[test]

397 + #[should_panic(expected = "configured test token id 256 was registered more than once")]

398 + fn configured_token_id_must_be_unique() {

399 + let _ = TestTokenizer::new()

400 + .with_regular_token("<token-a>", 256)

401 + .with_regular_token("<token-b>", 256);

402 + }

403 +

404 + #[test]

405 + fn unknown_decode_is_strict_by_default_and_configurable() {

406 + let strict = TestTokenizer::new();

407 + assert!(strict.decode(&[300], false).is_err());

408 + assert_eq!(

409 + TestTokenizer::new()

410 + .with_unknown_decode(UnknownDecode::Empty)

411 + .decode(&[b'a' as u32, 300, b'b' as u32], false)

412 + .unwrap(),

413 + "ab"

414 + );

415 + assert_eq!(

416 + TestTokenizer::new()

417 + .with_unknown_decode(UnknownDecode::Replacement)

418 + .decode(&[300], false)

419 + .unwrap(),

420 + "\u{FFFD}"

421 + );

422 + assert_eq!(strict.id_to_token(300), None);

423 + }

424 +

425 + #[test]

426 + fn explicit_vocab_size_overrides_inferred_bound() {

427 + let tokenizer = TestTokenizer::new()

428 + .with_regular_token("<high>", 10_000)

429 + .with_vocab_size(20_000);

430 +

431 + assert_eq!(tokenizer.vocab_size(), 20_000);

432 + assert_eq!(tokenizer.id_to_token(10_000).as_deref(), Some("<high>"));

433 + }

434 +}