A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 1, 21:15 UTC
[Bugfix][Rust Frontend] Fix UTF-8 char-boundary panic in incremental detokenizer (#44620)
Signed-off-by: Ting Sun <suntcrick@gmail.com>
Ting SUN committed
Jun 5, 2026 at 15:36 UTC
ca73293fa6f59cf58eda61ebd3f7117f097b4c7f
1 file changed
+25
rust/src/tokenizer/src/incremental.rs
+25
@@ -115,6 +115,8 @@ impl<T: Tokenizer + ?Sized> IncrementalDecoder for DecodeStream<'_, T> {
115
116
fn next_chunk(&mut self) -> Option<String> {
117
let cutoff = self.cumulative_output.len().saturating_sub(self.min_bytes_to_buffer);
118
+ // Ensure we split at a utf-8 char boundary.
119
+ let cutoff = self.cumulative_output.floor_char_boundary(cutoff);
120
(cutoff > self.output_index).then(|| {
121
let chunk = self.cumulative_output[self.output_index..cutoff].to_string();
122
self.output_index = cutoff;
@@ -356,4 +358,27 @@ mod tests {
358
assert_eq!(last_chunk.as_deref(), Some("lo!"));
359
assert_eq!(full_text, "Hello!");
360
}
361
+
362
+ #[test]
363
+ fn next_chunk_cutoff_respects_char_boundary() {
364
+ // Regression: next_chunk's cutoff (len - min_bytes_to_buffer) must be
365
+ // aligned to a UTF-8 char boundary like push_token/flush; otherwise
366
+ // streaming multi-byte output (CJK/emoji) with a hold-back buffer (set
367
+ // by a stop string) panics slicing cumulative_output mid-character.
368
+ let backend = Utf8Backend;
369
+ let mut decoder = backend.create_decode_stream(&[], false, 2);
370
+ let mut out = String::new();
371
+ for byte in "你好A".bytes() {
372
+ decoder.push_token(u32::from(byte)).unwrap();
373
+ if let Some(chunk) = decoder.next_chunk() {
374
+ out.push_str(&chunk);
375
+ }
376
+ }
377
+ let (last_chunk, full_text) = decoder.flush(None).unwrap();
378
+ if let Some(chunk) = last_chunk {
379
+ out.push_str(&chunk);
380
+ }
381
+ assert_eq!(full_text, "你好A");
382
+ assert_eq!(out, "你好A");
383
+ }
384
}