A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 09:15 UTC
Use FP32 for log probabilities (#19)
Woosuk Kwon committed
Mar 31, 2023 at 23:33 UTC
a90c97d72705f57b589062a2e09917dd9d27e389
1 file changed
+2
-1
cacheflow/models/sample.py
+2
-1
@@ -36,10 +36,11 @@ class Sampler(nn.Module):
36
# Use in-place division to avoid creating a new tensor.
37
logits.div_(t.unsqueeze(dim=1))
38
39
+ # We use float32 for probabilities and log probabilities.
40
# Compute the probabilities.
41
probs = torch.softmax(logits, dim=-1, dtype=torch.float)
42
# Compute the log probabilities (before applying top-p).
42
- logprobs = torch.log(probs, out=logits)
43
+ logprobs = torch.log(probs)
44
45
# Apply top-p truncation.
46
top_ps = _get_top_ps(input_metadata)