A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 17:00 UTC
[TPU][Bugfix] Fix next_token_ids shape (#8128)
Woosuk Kwon committed
Sep 3, 2024 at 13:29 UTC
0af3abe3d3225449c907d75eb3d2ae4b83bd21a1
1 file changed
+4
-1
vllm/worker/tpu_model_runner.py
+4
-1
@@ -601,7 +601,7 @@ class TPUModelRunner(ModelRunnerBase[ModelInputForTPU]):
601
batch_idx += 1
602
else:
603
for seq_id in seq_ids:
604
- next_token_id = next_token_ids[batch_idx][0]
604
+ next_token_id = next_token_ids[batch_idx]
605
seq_outputs.append(
606
SequenceOutput(seq_id, next_token_id,
607
{next_token_id: zero_logprob}))
@@ -722,6 +722,9 @@ class ModelWrapper(TorchCompileWrapperWithCustomDispacther):
722
sampled_token_ids = torch.multinomial(probs,
723
num_samples,
724
replacement=True)
725
+ if num_samples == 1:
726
+ argmax_token_ids = argmax_token_ids.squeeze(dim=-1)
727
+ sampled_token_ids = sampled_token_ids.squeeze(dim=-1)
728
next_token_ids = torch.where(t != 0, sampled_token_ids,
729
argmax_token_ids)
730
return next_token_ids