A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 18:15 UTC
Fix empty output when temp is too low (#2937)
Co-authored-by: Cyrus Leung <tlleungac@connect.ust.hk>
Chang Su committed
Aug 13, 2024 at 22:31 UTC
c134a4640214e760a8480b6d80b2038bf9ba3a8e
2 files changed
+9
-1
vllm/model_executor/layers/sampler.py
+2
-1
@@ -118,8 +118,9 @@ class Sampler(nn.Module):
118
sampling_tensors.frequency_penalties,
119
sampling_tensors.repetition_penalties)
120
121
- # Apply temperature scaling.
121
+ # Use float32 to apply temperature scaling.
122
# Use in-place division to avoid creating a new tensor.
123
+ logits = logits.to(torch.float)
124
logits.div_(sampling_tensors.temperatures.unsqueeze(dim=1))
125
126
if do_top_p_top_k:
vllm/sampling_params.py
+7
@@ -13,6 +13,7 @@ from vllm.logger import init_logger
13
logger = init_logger(__name__)
14
15
_SAMPLING_EPS = 1e-5
16
+_MAX_TEMP = 1e-2
17
18
19
class SamplingType(IntEnum):
@@ -145,6 +146,12 @@ class SamplingParams:
146
self.presence_penalty = presence_penalty
147
self.frequency_penalty = frequency_penalty
148
self.repetition_penalty = repetition_penalty
149
+ if 0 < temperature < _MAX_TEMP:
150
+ logger.warning(
151
+ "temperature %s is less than %s, which may cause numerical "
152
+ "errors nan or inf in tensors. We have maxed it out to %s.",
153
+ temperature, _MAX_TEMP, _MAX_TEMP)
154
+ temperature = max(temperature, _MAX_TEMP)
155
self.temperature = temperature
156
self.top_p = top_p
157
self.top_k = top_k