@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 0af3abe3d3
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 17:00 UTC

[TPU][Bugfix] Fix next_token_ids shape (#8128)

Woosuk Kwon committed Sep 3, 2024 at 13:29 UTC 0af3abe3d3225449c907d75eb3d2ae4b83bd21a1
1 file changed +4 -1
vllm/worker/tpu_model_runner.py
+4 -1
@@ -601,7 +601,7 @@ class TPUModelRunner(ModelRunnerBase[ModelInputForTPU]):
601 batch_idx += 1
602 else:
603 for seq_id in seq_ids:
604 - next_token_id = next_token_ids[batch_idx][0]
604 + next_token_id = next_token_ids[batch_idx]
605 seq_outputs.append(
606 SequenceOutput(seq_id, next_token_id,
607 {next_token_id: zero_logprob}))
@@ -722,6 +722,9 @@ class ModelWrapper(TorchCompileWrapperWithCustomDispacther):
722 sampled_token_ids = torch.multinomial(probs,
723 num_samples,
724 replacement=True)
725 + if num_samples == 1:
726 + argmax_token_ids = argmax_token_ids.squeeze(dim=-1)
727 + sampled_token_ids = sampled_token_ids.squeeze(dim=-1)
728 next_token_ids = torch.where(t != 0, sampled_token_ids,
729 argmax_token_ids)
730 return next_token_ids