A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 9, 23:00 UTC
[Bugfix] Fix byte fallback handling when using outlines (#31391)
Signed-off-by: Shinichi Hemmi <50256998+Alnusjaponica@users.noreply.github.com> Co-authored-by: Kenichi Maehashi <maehashi@preferred.jp>
Shinichi Hemmi committed
Jan 21, 2026 at 04:48 UTC
86c69dc54cc813b714afe8b123da64dcbbbd035e
2 files changed
+9
-2
vllm/v1/structured_output/backend_outlines.py
+6
-1
@@ -122,7 +122,12 @@ class OutlinesGrammar(StructuredOutputGrammar):
122
Returns False if the FSM failed to advance.
123
"""
124
if self.guide.accepts_tokens(tokens):
125
- # Advance cannot fail because we checked Guide.accepts_tokens()
125
+ # Advance can fail when the next state reached after advancing with
126
+ # the current tokens is a dead state. This is because Guide.accepts_tokens()
127
+ # only checks whether the current tokens can be accepted,
128
+ # whereas guide.advance() additionally checks the next state
129
+ # after all tokens are accepted.
130
+ # We need to be aware that the FSM must be prepared without dead states.
131
for t in tokens:
132
self.guide.advance(t)
133
self.num_processed_tokens += 1
vllm/v1/structured_output/utils.py
+3
-1
@@ -226,7 +226,9 @@ def _reduced_vocabulary(
226
# by this point.
227
token_bytes = bytes(token_str) # type: ignore[arg-type]
228
229
- elif "\ufffd" in token_str and not re_replacement_seq.match(token_str):
229
+ elif (token_str == "\ufffd" and token != "\ufffd") or (
230
+ "\ufffd" in token_str and not re_replacement_seq.match(token_str)
231
+ ):
232
# Handle tokens with invalid UTF-8 sequences.
233
if re_llama_byte_token.match(token):
234
# Llama-like tokenizers use <0xXX> for incomplete sequences.