A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[bugfix] fix structured outputs key missing issue from #24929 (#25195)
Signed-off-by: Lu Fang <fanglu@fb.com>
Lucia Fang committed
Sep 19, 2025 at 11:37 UTC
b716ab93a781156ab178513afc0e407cc72d443b
2 files changed
+9
-6
vllm/v1/core/sched/scheduler.py
+3
-1
@@ -578,8 +578,10 @@ class Scheduler(SchedulerInterface):
578
scheduled_spec_decode_tokens,
579
req_to_new_blocks,
580
)
581
+ scheduled_requests = (scheduled_new_reqs + scheduled_running_reqs +
582
+ scheduled_resumed_reqs)
583
structured_output_request_ids, grammar_bitmask = (
582
- self.get_grammar_bitmask(self.running,
584
+ self.get_grammar_bitmask(scheduled_requests,
585
scheduled_spec_decode_tokens))
586
scheduler_output = SchedulerOutput(
587
scheduled_new_reqs=new_reqs_data,
vllm/v1/structured_output/utils.py
+6
-5
@@ -90,13 +90,14 @@ def apply_grammar_bitmask(
90
seq = sorted(scheduler_output.structured_output_request_ids.items(),
91
key=lambda x: x[1])
92
for req_id, _ in seq:
93
- logit_index = struct_out_req_batch_indices[req_id]
93
num_spec_tokens = len(
94
scheduler_output.scheduled_spec_decode_tokens.get(req_id, []))
96
- for i in range(1 + num_spec_tokens):
97
- sorted_bitmask[logit_index + i] = \
98
- grammar_bitmask[cumulative_index + i]
99
- out_indices.append(logit_index + i)
95
+ if req_id in struct_out_req_batch_indices:
96
+ logit_index = struct_out_req_batch_indices[req_id]
97
+ for i in range(1 + num_spec_tokens):
98
+ sorted_bitmask[logit_index + i] = \
99
+ grammar_bitmask[cumulative_index + i]
100
+ out_indices.append(logit_index + i)
101
cumulative_index += 1 + num_spec_tokens
102
grammar_bitmask = sorted_bitmask
103