[Bugfix][Rust Frontend] Skip undefined token ids in decode and anchor them zero-width (#55240)
Co-authored-by: Bugen Zhao <i@bugenzhao.com> Co-authored-by: Codex <noreply@openai.com> Signed-off-by: Feathbow <feathbow@gmail.com> Signed-off-by: Bugen Zhao <i@bugenzhao.com>
Salt Sato committed
Sep 9, 2026 at 10:55 UTC
c7e9816c6ab0731165a134fd0a9defed6ab1d748
2 files changed
+68
-26
rust/src/tokenizer/src/hf.rs
+49
-9
@@ -54,11 +54,10 @@ fn decode_fastokens_byte_level(
54
let tokens: Vec<&str> = token_ids
55
.iter()
56
.filter(|&&id| !(skip_special_tokens && t.is_special_token(id)))
57
- .map(|&id| {
58
- t.id_to_token(id)
59
- .ok_or_else(|| tokenizer_error!("decoding failed: unknown token ID: {id}"))
60
- })
61
- .collect::<Result<_>>()?;
57
+ // Match HF and fastokens: model vocabularies may contain undefined
58
+ // tokenizer IDs, which contribute no decoded text.
59
+ .filter_map(|&id| t.id_to_token(id))
60
+ .collect();
61
Ok(decode_byte_level(tokens))
62
}
63
@@ -337,6 +336,7 @@ mod tests {
336
use tokenizers::{AddedToken, Tokenizer as HfTokenizer};
337
338
use super::{HuggingFaceTokenizer, Tokenizer};
339
+ use crate::{TokenAnchor, TokenAttribution};
340
341
const REGULAR_TOKEN: &str = "<|regular|>";
342
const SPECIAL_TOKEN: &str = "<|special|>";
@@ -717,10 +717,50 @@ mod tests {
717
}
718
719
#[test]
720
- fn fast_byte_level_errors_on_unknown_id() {
720
+ fn fast_byte_level_skips_undefined_ids() {
721
let t = tiny_byte_level_bpe();
722
- let err = super::decode_fastokens_byte_level(&t, &[999], false)
723
- .expect_err("unknown id must error");
724
- assert!(format!("{err:?}").contains("999"));
722
+ assert_eq!(
723
+ super::decode_fastokens_byte_level(&t, &[1, 2, 999, 3, 3, 4], false)
724
+ .expect("with the id"),
725
+ super::decode_fastokens_byte_level(&t, &[1, 2, 3, 3, 4], false)
726
+ .expect("without the id")
727
+ );
728
+ }
729
+
730
+ #[test]
731
+ fn decode_stream_anchors_undefined_ids_zero_width() {
732
+ let wrapper = HuggingFaceTokenizer::from_fastokens_backend(tiny_byte_level_bpe());
733
+ assert!(matches!(
734
+ wrapper.backend,
735
+ super::Backend::FastokensByteLevel(_)
736
+ ));
737
+
738
+ let mut stream = wrapper.create_decode_stream(&[], false, 0);
739
+ for id in [999, 1, 999, 2] {
740
+ stream.push_token(id).expect("push token");
741
+ }
742
+ let (_, full) = stream.flush(None).expect("flush");
743
+ assert_eq!(full.text, "He");
744
+ assert_eq!(
745
+ full.attributions.as_slice(),
746
+ [
747
+ TokenAttribution {
748
+ token_id: 999,
749
+ anchor: TokenAnchor::ZeroWidth { byte_offset: 0 }
750
+ },
751
+ TokenAttribution {
752
+ token_id: 1,
753
+ anchor: TokenAnchor::Visible { byte_offset: 0 }
754
+ },
755
+ TokenAttribution {
756
+ token_id: 999,
757
+ anchor: TokenAnchor::ZeroWidth { byte_offset: 1 }
758
+ },
759
+ TokenAttribution {
760
+ token_id: 2,
761
+ anchor: TokenAnchor::Visible { byte_offset: 1 }
762
+ },
763
+ ]
764
+ );
765
}
766
}
rust/src/tokenizer/src/incremental.rs
+19
-17
@@ -72,21 +72,11 @@ const SAFE_SUFFIX_MIN: usize = 4;
72
const SAFE_SUFFIX_MAX: usize = 6;
73
74
impl<T: Tokenizer + ?Sized> DecodeStream<'_, T> {
75
- fn record_token(&mut self, token_id: u32) {
76
- if self.skip_special_tokens && self.tokenizer.is_special_id(token_id) {
77
- self.decoded.record_zero_width_token(token_id);
78
- } else {
79
- self.decoded.record_pending_token(token_id);
80
- }
81
- }
82
-
75
/// Decode prompt-only context for prefix seeding.
76
///
77
/// Prompt ids may come from the model vocabulary rather than the local
78
/// tokenizer vocabulary. For this seeding path, ids that cannot be mapped
87
- /// back to raw token text are dropped before retrying strict decode. This
88
- /// tolerance is intentionally limited to prompt context; generated ids are
89
- /// decoded later through the normal strict path.
79
+ /// back to raw token text are dropped before retrying strict decode.
80
fn decode_prompt_context(&self, ids: &[u32]) -> Result<(String, Vec<u32>)> {
81
match self.tokenizer.decode(ids, self.skip_special_tokens) {
82
Ok(decoded) => Ok((decoded, ids.to_vec())),
@@ -144,10 +134,22 @@ impl<T: Tokenizer + ?Sized> IncrementalDecoder for DecodeStream<'_, T> {
134
}
135
136
self.ids.push(token_id);
147
- self.record_token(token_id);
137
let string = self.tokenizer.decode(&self.ids, self.skip_special_tokens)?;
138
let prefix_len = self.prefix.len();
150
- if string.len() <= prefix_len || string.ends_with('\u{FFFD}') {
139
+ let produced_text = string.len() > prefix_len && !string.ends_with('\u{FFFD}');
140
+
141
+ // Skipped undefined IDs leave the decoded text unchanged. Check only
142
+ // non-emitting steps to avoid id_to_token's lookup and String allocation
143
+ // on the normal path.
144
+ let zero_width = (self.skip_special_tokens && self.tokenizer.is_special_id(token_id))
145
+ || (!produced_text && self.tokenizer.id_to_token(token_id).is_none());
146
+ if zero_width {
147
+ self.decoded.record_zero_width_token(token_id);
148
+ } else {
149
+ self.decoded.record_pending_token(token_id);
150
+ }
151
+
152
+ if !produced_text {
153
return Ok(0);
154
}
155
// Ensure we split at a utf-8 char boundary.
@@ -224,8 +226,8 @@ mod tests {
226
unreachable!()
227
}
228
227
- fn id_to_token(&self, _id: u32) -> Option<String> {
228
- unreachable!()
229
+ fn id_to_token(&self, id: u32) -> Option<String> {
230
+ Some(id.to_string())
231
}
232
}
233
@@ -456,8 +458,8 @@ mod tests {
458
unreachable!()
459
}
460
459
- fn id_to_token(&self, _id: u32) -> Option<String> {
460
- unreachable!()
461
+ fn id_to_token(&self, id: u32) -> Option<String> {
462
+ Some(id.to_string())
463
}
464
}
465