A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[Structured Outputs] [Bug] Fix misalignment in apply_grammar_bitmask causing unintended masking and NaN logits (#22963)
Signed-off-by: rishitdholakia13 <rishit+github@cohere.com>
rishitdholakia13 committed
Aug 15, 2025 at 17:25 UTC
1fc375dc053424c3b8656802d35d5251e75fc857
1 file changed
+2
-2
vllm/v1/worker/gpu_model_runner.py
+2
-2
@@ -1355,10 +1355,10 @@ class GPUModelRunner(LoRAModelRunnerMixin, KVConnectorModelRunnerMixin):
1355
cumulative_index += 1 + num_spec_tokens
1356
grammar_bitmask = sorted_bitmask
1357
1358
- # If the grammar bitmask and the logits have the same shape
1358
+ # If the length of out indices and the logits have the same shape
1359
# we don't need to pass indices to the kernel,
1360
# since the bitmask is already aligned with the logits.
1361
- skip_out_indices = grammar_bitmask.shape[0] == logits.shape[0]
1361
+ skip_out_indices = len(out_indices) == logits.shape[0]
1362
1363
# Serialization of np.ndarray is much more efficient than a tensor,
1364
# so we receive it in that format.