A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 11, 18:45 UTC
[Model Runner V2] Fix Triton warning on tl.where (#30355)
Signed-off-by: Woosuk Kwon <woosuk.kwon@berkeley.edu>
Woosuk Kwon committed
Dec 9, 2025 at 09:59 UTC
9e6562a3f625279fd7c8b9ac53c30fed3b01f5b9
1 file changed
+1
vllm/v1/worker/gpu/sample/penalties.py
+1
@@ -62,6 +62,7 @@ def _penalties_and_temperature_kernel(
62
mask=packed_block < tl.cdiv(vocab_size, 32),
63
)
64
prompt_bin_mask = (packed_mask[:, None] >> (tl.arange(0, 32)[None, :])) & 1
65
+ prompt_bin_mask = prompt_bin_mask.to(tl.int1)
66
prompt_bin_mask = prompt_bin_mask.reshape(BLOCK_SIZE)
67
68
# If token appears in prompt or output, apply, otherwise use 1.0 for no-op.