A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[FIXBUG] Correctly Apply Grammar Bitmask in Mixed Batches (#22896)
Signed-off-by: JartX <sagformas@epdcenter.es>
JartX committed
Aug 15, 2025 at 19:42 UTC
68af77e51c5ca78ec0fd2496eca80b2257176b6e
1 file changed
+4
-3
vllm/v1/worker/gpu_model_runner.py
+4
-3
@@ -1337,9 +1337,10 @@ class GPUModelRunner(LoRAModelRunnerMixin, KVConnectorModelRunnerMixin):
1337
out_indices = []
1338
1339
# Reorder the bitmask to match the order of the requests in the batch.
1340
- sorted_bitmask = np.zeros_like(grammar_bitmask,
1341
- shape=(logits.shape[0],
1342
- grammar_bitmask.shape[1]))
1340
+ sorted_bitmask = np.full(shape=(logits.shape[0],
1341
+ grammar_bitmask.shape[1]),
1342
+ fill_value=-1,
1343
+ dtype=grammar_bitmask.dtype)
1344
cumulative_index = 0
1345
seq = sorted(scheduler_output.structured_output_request_ids.items(),
1346
key=lambda x: x[1])