@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / c134a46402
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 18:15 UTC

Fix empty output when temp is too low (#2937)

Co-authored-by: Cyrus Leung <tlleungac@connect.ust.hk>

Chang Su committed Aug 13, 2024 at 22:31 UTC c134a4640214e760a8480b6d80b2038bf9ba3a8e
2 files changed +9 -1
vllm/model_executor/layers/sampler.py
+2 -1
@@ -118,8 +118,9 @@ class Sampler(nn.Module):
118 sampling_tensors.frequency_penalties,
119 sampling_tensors.repetition_penalties)
120
121 - # Apply temperature scaling.
121 + # Use float32 to apply temperature scaling.
122 # Use in-place division to avoid creating a new tensor.
123 + logits = logits.to(torch.float)
124 logits.div_(sampling_tensors.temperatures.unsqueeze(dim=1))
125
126 if do_top_p_top_k:
vllm/sampling_params.py
+7
@@ -13,6 +13,7 @@ from vllm.logger import init_logger
13 logger = init_logger(__name__)
14
15 _SAMPLING_EPS = 1e-5
16 +_MAX_TEMP = 1e-2
17
18
19 class SamplingType(IntEnum):
@@ -145,6 +146,12 @@ class SamplingParams:
146 self.presence_penalty = presence_penalty
147 self.frequency_penalty = frequency_penalty
148 self.repetition_penalty = repetition_penalty
149 + if 0 < temperature < _MAX_TEMP:
150 + logger.warning(
151 + "temperature %s is less than %s, which may cause numerical "
152 + "errors nan or inf in tensors. We have maxed it out to %s.",
153 + temperature, _MAX_TEMP, _MAX_TEMP)
154 + temperature = max(temperature, _MAX_TEMP)
155 self.temperature = temperature
156 self.top_p = top_p
157 self.top_k = top_k