A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 07:15 UTC
[Bugfix]: Fix the incompatibility issue with Structured Outputs when Thinking is disabled (#18879)
Signed-off-by: chaunceyjiang <chaunceyjiang@gmail.com>
Chauncey committed
May 31, 2025 at 17:20 UTC
ba5111f2372678fb03e96ee69decd0febd03f13e
2 files changed
+19
-13
vllm/v1/structured_output/__init__.py
+18
-12
@@ -149,31 +149,37 @@ class StructuredOutputManager:
149
# NOTE: This outer loop can likely be parallelized to improve
150
# performance of bitmask generation for large batches.
151
for req_id, _ in ordered_seq:
152
- request = requests[req_id].structured_output_request
153
- if TYPE_CHECKING:
154
- assert request is not None
155
- assert request.grammar is not None
152
+ request = requests[req_id]
153
+ structured_output_request = request.structured_output_request
154
157
- apply_bitmask = (
158
- request.reasoning_ended if self.reasoner is not None else True
159
- ) # noqa: E501
155
+ if TYPE_CHECKING:
156
+ assert structured_output_request is not None
157
+ assert structured_output_request.grammar is not None
158
+ apply_bitmask: bool = True
159
+ if self.reasoner is not None:
160
+ if structured_output_request.reasoning_ended is None:
161
+ structured_output_request.reasoning_ended = \
162
+ self.reasoner.is_reasoning_end(request.prompt_token_ids)
163
+ apply_bitmask = structured_output_request.reasoning_ended
164
165
state_advancements = 0
166
req_tokens = scheduled_spec_decode_tokens.get(req_id, []) + [None]
167
for i, token in enumerate(req_tokens):
164
- if apply_bitmask and not request.grammar.is_terminated():
165
- request.grammar.fill_bitmask(bitmask_tensor,
166
- cumulative_index)
168
+ if apply_bitmask and not \
169
+ structured_output_request.grammar.is_terminated():
170
+ structured_output_request.grammar.fill_bitmask(
171
+ bitmask_tensor, cumulative_index)
172
if token is not None:
173
# In order to generate the correct bitmask for each
174
# position in the speculative sequence, we advance
175
# the FSM state for each speculative token and rollback
176
# to restore the previous state when we are finished.
172
- assert request.grammar.accept_tokens(req_id, [token])
177
+ assert structured_output_request.grammar.accept_tokens(
178
+ req_id, [token])
179
state_advancements += 1
180
cumulative_index += 1
181
if state_advancements > 0:
176
- request.grammar.rollback(state_advancements)
182
+ structured_output_request.grammar.rollback(state_advancements)
183
184
if cumulative_index < bitmask_tensor.shape[0]:
185
bitmask_tensor = bitmask_tensor[:cumulative_index]
vllm/v1/structured_output/request.py
+1
-1
@@ -20,7 +20,7 @@ class StructuredOutputRequest:
20
sampling_params: SamplingParams
21
_grammar: Optional[Union[Future[StructuredOutputGrammar],
22
StructuredOutputGrammar]] = None
23
- reasoning_ended: bool = False
23
+ reasoning_ended: Optional[bool] = None
24
25
def _check_grammar_completion(self) -> bool:
26
# NOTE: We have to lazy import to gate circular imports