A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 09:45 UTC
[Bugfix] Use random hidden states in dummy sampler run (#18543)
Signed-off-by: Bowen Wang <abmfy@icloud.com>
Bowen Wang committed
May 22, 2025 at 06:48 UTC
4e04eceb58288310932d4abfbb417f1415d05caf
1 file changed
+4
vllm/v1/worker/gpu_model_runner.py
+4
@@ -1721,6 +1721,10 @@ class GPUModelRunner(LoRAModelRunnerMixin):
1721
self,
1722
hidden_states: torch.Tensor,
1723
) -> torch.Tensor:
1724
+ # The dummy hidden states may contain special values,
1725
+ # like `inf` or `nan`.
1726
+ # To avoid breaking the sampler, we use a random tensor here instead.
1727
+ hidden_states = torch.rand_like(hidden_states)
1728
1729
logits = self.model.compute_logits(hidden_states, None)
1730
num_reqs = logits.size(0)