A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 22:00 UTC
[Bugfix][TPU] Fix pad slot id (#5977)
Woosuk Kwon committed
Jun 28, 2024 at 18:55 UTC
7f83f40dee2e92ff005d44a262a3cf42c87c2082
1 file changed
+1
-1
vllm/worker/tpu_model_runner.py
+1
-1
@@ -19,7 +19,7 @@ from vllm.utils import make_tensor_with_pad
19
20
logger = init_logger(__name__)
21
22
-_PAD_SLOT_ID = 0 # FIXME(woosuk)
22
+_PAD_SLOT_ID = -1 # NOTE(woosuk): In PyTorch XLA, index -1 is ignored.
23
# FIXME(woosuk): Temporarily disabled top-p sampling since it's too slow.
24
_ENABLE_TOP_P = False
25
# FIXME(woosuk): A temporary hack to support `n > 1`.