@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 6e063ea35b
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 19:30 UTC

[TPU] Fix greedy decoding (#6933)

Woosuk Kwon committed Jul 30, 2024 at 02:06 UTC 6e063ea35b6fe3b6a9b87aae589725493849a632
1 file changed +18 -9
vllm/worker/tpu_model_runner.py
+18 -9
@@ -28,7 +28,9 @@ if TYPE_CHECKING:
28
29 logger = init_logger(__name__)
30
31 -_PAD_SLOT_ID = -1 # NOTE(woosuk): In PyTorch XLA, index -1 is ignored.
31 +# Here we utilize the behavior that out-of-bound index is ignored.
32 +# FIXME(woosuk): Find a more reliable way to prevent possible bugs.
33 +_PAD_SLOT_ID = 1_000_000_000
34 # FIXME(woosuk): Temporarily disabled top-p sampling since it's too slow.
35 _ENABLE_TOP_P = False
36 # FIXME(woosuk): A temporary hack to support `n > 1`.
@@ -414,10 +416,7 @@ class TPUModelRunner(ModelRunnerBase[ModelInputForTPU]):
416 best_of = []
417 for seq_group_metadata in seq_group_metadata_list:
418 sampling_params = seq_group_metadata.sampling_params
417 - # NOTE(woosuk): Here we mimic argmax sampling by applying a very
418 - # low temperature. This is not accurate.
419 - t.append(sampling_params.temperature
420 - if sampling_params.temperature >= 1e-5 else 1e-5)
419 + t.append(sampling_params.temperature)
420 if sampling_params.top_p != 1 and not _ENABLE_TOP_P:
421 raise NotImplementedError(
422 "Top-p sampling is currently disabled for the TPU backend "
@@ -678,13 +677,23 @@ class ModelWrapper(nn.Module):
677 hidden_states = hidden_states.flatten(0, 1)
678 logits = self.model.compute_logits(hidden_states, sampling_metadata)
679
681 - logits = logits / t.unsqueeze(dim=1)
680 + # Argmax sampling.
681 + argmax_token_ids = torch.argmax(logits, dim=-1, keepdim=True)
682 + argmax_token_ids = argmax_token_ids.repeat(1, num_samples)
683 +
684 + # Zero temperature means greedy decoding. Avoid division by zero.
685 + nonzero_t = torch.where(t != 0, t, 1.0)
686 + logits = logits / nonzero_t.unsqueeze(dim=1)
687 if _ENABLE_TOP_P:
688 logits = _apply_top_p(logits, p.unsqueeze(dim=1))
689 +
690 + # Random sampling.
691 probs = torch.softmax(logits, dim=-1, dtype=torch.float32)
685 - next_token_ids = torch.multinomial(probs,
686 - num_samples,
687 - replacement=True)
692 + sampled_token_ids = torch.multinomial(probs,
693 + num_samples,
694 + replacement=True)
695 + next_token_ids = torch.where(t != 0, sampled_token_ids,
696 + argmax_token_ids)
697 return next_token_ids
698
699