A high-throughput and memory-efficient inference and serving engine for LLMs
[Bugfix][Rust Frontend] Fix missing added tokens in hf/fastokens tokenizer (#44683)
Signed-off-by: Isotr0py <mozf@mail2.sysu.edu.cn> Signed-off-by: Bugen Zhao <i@bugenzhao.com> Co-authored-by: Bugen Zhao <i@bugenzhao.com>
Isotr0py committed
Jun 10, 2026 at 18:52 UTC
9ad08c4d1513af4c2df34a20aa02dc0da688042d
4 files changed
+215
-8
rust/src/chat/src/multimodal.rs
+1
-1
@@ -225,7 +225,7 @@ impl MultimodalModelInfo {
225
///
226
/// The HF renderer uses this token while flattening image content in string
227
/// content format.
228
- pub(crate) fn placeholder_token(&self) -> &str {
228
+ pub fn placeholder_token(&self) -> &str {
229
&self.spec.placeholder_token
230
}
231
}
rust/src/server/src/routes/tests.rs
+18
-5
@@ -427,6 +427,11 @@ impl Tokenizer for FakeChatTokenizer {
427
rest = stripped;
428
continue;
429
}
430
+ if let Some(stripped) = rest.strip_prefix("<|image_pad|>") {
431
+ token_ids.push(151655);
432
+ rest = stripped;
433
+ continue;
434
+ }
435
436
let ch = rest.chars().next().expect("rest is not empty");
437
let mut buf = [0; 4];
@@ -547,11 +552,16 @@ impl ChatBackend for FakeChatBackend {
552
553
impl ChatRenderer for FakeChatBackend {
554
fn render(&self, request: &ChatRequest) -> vllm_chat::Result<vllm_chat::RenderedPrompt> {
555
+ let placeholder = self
556
+ .multimodal_model_info
557
+ .as_ref()
558
+ .map(|info| info.placeholder_token())
559
+ .unwrap_or("<image>");
560
let mut prompt = String::new();
561
for message in &request.messages {
562
prompt.push_str(message.role().as_str());
563
prompt.push_str(": ");
554
- prompt.push_str(&render_fake_message_content(message)?);
564
+ prompt.push_str(&render_fake_message_content(message, placeholder)?);
565
prompt.push('\n');
566
}
567
if request.chat_options.add_generation_prompt() {
@@ -563,17 +573,20 @@ impl ChatRenderer for FakeChatBackend {
573
}
574
}
575
566
-fn render_fake_message_content(message: &ChatMessage) -> vllm_chat::Result<String> {
576
+fn render_fake_message_content(
577
+ message: &ChatMessage,
578
+ placeholder: &str,
579
+) -> vllm_chat::Result<String> {
580
match message {
581
ChatMessage::System { content }
582
| ChatMessage::Developer { content, .. }
583
| ChatMessage::User { content }
571
- | ChatMessage::ToolResponse { content, .. } => render_fake_content(content),
584
+ | ChatMessage::ToolResponse { content, .. } => render_fake_content(content, placeholder),
585
ChatMessage::Assistant { .. } => message.text_content(),
586
}
587
}
588
576
-fn render_fake_content(content: &ChatContent) -> vllm_chat::Result<String> {
589
+fn render_fake_content(content: &ChatContent, placeholder: &str) -> vllm_chat::Result<String> {
590
Ok(match content {
591
ChatContent::Text(text) => text.clone(),
592
ChatContent::Parts(parts) => {
@@ -581,7 +594,7 @@ fn render_fake_content(content: &ChatContent) -> vllm_chat::Result<String> {
594
for part in parts {
595
match part {
596
ChatContentPart::Text { text } => out.push_str(text),
584
- ChatContentPart::ImageUrl { .. } => out.push_str("<image>"),
597
+ ChatContentPart::ImageUrl { .. } => out.push_str(placeholder),
598
}
599
}
600
out
rust/src/tokenizer/src/hf.rs
+38
-2
@@ -8,8 +8,11 @@ use tokenizers::Tokenizer as HfTokenizer;
8
use tracing::{info, warn};
9
10
use crate::byte_level_decode::decode_byte_level;
11
+use crate::hf::added_tokens::load_tokenizer_json_with_extra_tokens;
12
use crate::{Result, Tokenizer};
13
14
+mod added_tokens;
15
+
16
enum Backend {
17
Hf(Box<HfTokenizer>),
18
Fastokens(Box<FastokensTokenizer>),
@@ -104,7 +107,8 @@ impl HuggingFaceTokenizer {
107
/// Load from `tokenizer.json` with `fastokens`.
108
pub fn new_fastokens(path: &Path) -> Result<Self> {
109
info!(path = %path.display(), "loading tokenizer with fastokens");
107
- let t = FastokensTokenizer::from_file(path)
110
+ let tokenizer_json = load_tokenizer_json_with_extra_tokens(path)?;
111
+ let t = FastokensTokenizer::from_json(tokenizer_json)
112
.map_err(|error| tokenizer_error!("failed to load tokenizer: {}", error.as_report()))?;
113
Ok(Self::from_fastokens_backend(t))
114
}
@@ -112,7 +116,8 @@ impl HuggingFaceTokenizer {
116
/// Load from `tokenizer.json` with Hugging Face `tokenizers`.
117
pub fn new_hf(path: &Path) -> Result<Self> {
118
info!(path = %path.display(), "loading tokenizer with huggingface tokenizers");
115
- let t = HfTokenizer::from_file(path)
119
+ let tokenizer_json = load_tokenizer_json_with_extra_tokens(path)?;
120
+ let t = serde_json::from_value::<HfTokenizer>(tokenizer_json)
121
.map_err(|error| tokenizer_error!("failed to load tokenizer: {}", error.as_report()))?;
122
Ok(Self::from_hf_backend(t))
123
}
@@ -250,6 +255,37 @@ mod tests {
255
assert!(wrapper.is_special_id(special_id));
256
}
257
258
+ #[test]
259
+ fn constructors_merge_extra_added_tokens_from_tokenizer_config() {
260
+ let tokenizer = tiny_bpe_tokenizer();
261
+
262
+ let dir = tempdir().expect("create temp dir");
263
+ let path = dir.path().join("tokenizer.json");
264
+ tokenizer.save(&path, false).expect("save tokenizer json");
265
+ std::fs::write(
266
+ dir.path().join("tokenizer_config.json"),
267
+ r#"{
268
+ "added_tokens_decoder": {
269
+ "9": {
270
+ "content": "<|image_pad|>",
271
+ "special": true,
272
+ "normalized": false
273
+ }
274
+ }
275
+ }"#,
276
+ )
277
+ .expect("write tokenizer config");
278
+
279
+ for wrapper in [
280
+ HuggingFaceTokenizer::new_fastokens(&path).expect("load fastokens wrapper"),
281
+ HuggingFaceTokenizer::new_hf(&path).expect("load hf wrapper"),
282
+ ] {
283
+ assert_eq!(wrapper.token_to_id("<|image_pad|>"), Some(9));
284
+ assert_eq!(wrapper.id_to_token(9).as_deref(), Some("<|image_pad|>"));
285
+ assert!(wrapper.is_special_id(9));
286
+ }
287
+ }
288
+
289
/// BPE tokenizer that round-trips through fastokens with a genuine
290
/// `ByteLevel` decoder; vocab covers both GPT-2 (Ġ U+0120) and non-GPT-2
291
/// (｜ U+FF5C) codepoints.
rust/src/tokenizer/src/hf/added_tokens.rs
new
+158
@@ -0,0 +1,158 @@
1
+use serde::{Deserialize, Serialize};
2
+use thiserror_ext::AsReport as _;
3
+use tracing::warn;
4
+
5
+use crate::Result;
6
+
7
+use std::{fs, path::Path};
8
+
9
+/// Minimal `tokenizer.json` projection used to patch `added_tokens` while
10
+/// preserving the rest of the tokenizer definition verbatim.
11
+#[derive(Debug, Deserialize, Serialize)]
12
+struct TokenizerJson {
13
+ #[serde(default)]
14
+ added_tokens: Vec<AddedTokenConfig>,
15
+ #[serde(flatten)]
16
+ extra: serde_json::Map<String, serde_json::Value>,
17
+}
18
+
19
+/// Minimal `tokenizer_config.json` projection for Hugging Face's
20
+/// `added_tokens_decoder` map. Other config keys are intentionally ignored.
21
+#[derive(Debug, Deserialize)]
22
+struct TokenizerConfigJson {
23
+ #[serde(default)]
24
+ added_tokens_decoder: std::collections::HashMap<String, AddedTokenConfig>,
25
+}
26
+
27
+/// Hugging Face added-token payload. `tokenizer.json` stores `id` inside each
28
+/// item, while `tokenizer_config.json` stores it as the map key.
29
+#[derive(Clone, Debug, Deserialize, Serialize)]
30
+struct AddedTokenConfig {
31
+ #[serde(default, skip_serializing_if = "Option::is_none")]
32
+ id: Option<u32>,
33
+ content: String,
34
+ #[serde(default)]
35
+ single_word: bool,
36
+ #[serde(default)]
37
+ lstrip: bool,
38
+ #[serde(default)]
39
+ rstrip: bool,
40
+ #[serde(default)]
41
+ normalized: bool,
42
+ #[serde(default)]
43
+ special: bool,
44
+}
45
+
46
+impl AddedTokenConfig {
47
+ /// Return this added-token payload in `tokenizer.json` shape by filling the
48
+ /// numeric token id that came from `added_tokens_decoder`'s string key.
49
+ fn with_id(mut self, id: u32) -> Self {
50
+ self.id = Some(id);
51
+ self
52
+ }
53
+}
54
+
55
+/// Read `tokenizer.json`, then merge in extra added tokens from `tokenizer_config.json`.
56
+pub(super) fn load_tokenizer_json_with_extra_tokens(path: &Path) -> Result<serde_json::Value> {
57
+ let tokenizer_json = fs::read_to_string(path)
58
+ .map_err(|error| tokenizer_error!("failed to read {}: {}", path.display(), error))?;
59
+ let mut tokenizer_json: TokenizerJson = serde_json::from_str(&tokenizer_json)
60
+ .map_err(|error| tokenizer_error!("failed to parse {}: {}", path.display(), error))?;
61
+
62
+ if let Some(parent) = path.parent() {
63
+ let config_path = parent.join("tokenizer_config.json");
64
+ if config_path.exists() {
65
+ match load_tokenizer_config_json(&config_path) {
66
+ Ok(config_json) => merge_added_tokens_from_config(&mut tokenizer_json, config_json),
67
+ Err(error) => {
68
+ warn!(
69
+ path = %config_path.display(),
70
+ error = %error.as_report(),
71
+ "failed to load tokenizer_config.json; skipping extra added tokens"
72
+ );
73
+ }
74
+ }
75
+ }
76
+ }
77
+
78
+ serde_json::to_value(tokenizer_json)
79
+ .map_err(|error| tokenizer_error!("failed to serialize tokenizer json: {}", error))
80
+}
81
+
82
+/// Read and parse a sibling `tokenizer_config.json`.
83
+fn load_tokenizer_config_json(path: &Path) -> Result<TokenizerConfigJson> {
84
+ let text = fs::read_to_string(path)
85
+ .map_err(|error| tokenizer_error!("failed to read {}: {}", path.display(), error))?;
86
+ serde_json::from_str(&text)
87
+ .map_err(|error| tokenizer_error!("failed to parse {}: {}", path.display(), error))
88
+}
89
+
90
+/// Merge added_tokens in `tokenizer.json` and `tokenizer_config.json`.
91
+fn merge_added_tokens_from_config(
92
+ tokenizer_json: &mut TokenizerJson,
93
+ config_json: TokenizerConfigJson,
94
+) {
95
+ use std::collections::HashSet;
96
+
97
+ let mut existing_ids: HashSet<u32> =
98
+ tokenizer_json.added_tokens.iter().filter_map(|token| token.id).collect();
99
+
100
+ let mut extra_tokens = Vec::with_capacity(config_json.added_tokens_decoder.len());
101
+ for (id_str, token) in config_json.added_tokens_decoder {
102
+ let id = match id_str.parse::<u32>() {
103
+ Ok(id) => id,
104
+ Err(_) => continue,
105
+ };
106
+ extra_tokens.push((id, token));
107
+ }
108
+ extra_tokens.sort_unstable_by_key(|(id, _)| *id);
109
+
110
+ for (id, token) in extra_tokens {
111
+ if existing_ids.contains(&id) {
112
+ continue;
113
+ }
114
+
115
+ // Convert from decoder format to added_tokens array format by adding the "id" field.
116
+ tokenizer_json.added_tokens.push(token.with_id(id));
117
+ existing_ids.insert(id);
118
+ }
119
+}
120
+
121
+#[cfg(test)]
122
+mod tests {
123
+ use super::*;
124
+
125
+ #[test]
126
+ fn merge_added_tokens_from_config_preserves_unmodeled_fields() {
127
+ let mut tokenizer_json: TokenizerJson = serde_json::from_value(serde_json::json!({
128
+ "version": "1.0",
129
+ "added_tokens": [
130
+ {"id": 0, "content": "<unk>", "special": true}
131
+ ],
132
+ "model": {"type": "WordLevel"}
133
+ }))
134
+ .expect("parse tokenizer json");
135
+
136
+ let config_json: TokenizerConfigJson = serde_json::from_value(serde_json::json!({
137
+ "chat_template": "{{ messages }}",
138
+ "added_tokens_decoder": {
139
+ "1": {
140
+ "content": "<|image_pad|>",
141
+ "special": true,
142
+ "normalized": false
143
+ }
144
+ }
145
+ }))
146
+ .expect("parse tokenizer config");
147
+
148
+ merge_added_tokens_from_config(&mut tokenizer_json, config_json);
149
+ let merged = serde_json::to_value(tokenizer_json).expect("serialize tokenizer json");
150
+
151
+ assert_eq!(merged["version"], "1.0");
152
+ assert_eq!(merged["model"]["type"], "WordLevel");
153
+ assert_eq!(merged["added_tokens"][1]["id"], 1);
154
+ assert_eq!(merged["added_tokens"][1]["content"], "<|image_pad|>");
155
+ assert_eq!(merged["added_tokens"][1]["special"], true);
156
+ assert_eq!(merged["added_tokens"][1]["normalized"], false);
157
+ }
158
+}