@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / a90c97d727
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 09:15 UTC

Use FP32 for log probabilities (#19)

Woosuk Kwon committed Mar 31, 2023 at 23:33 UTC a90c97d72705f57b589062a2e09917dd9d27e389
1 file changed +2 -1
cacheflow/models/sample.py
+2 -1
@@ -36,10 +36,11 @@ class Sampler(nn.Module):
36 # Use in-place division to avoid creating a new tensor.
37 logits.div_(t.unsqueeze(dim=1))
38
39 + # We use float32 for probabilities and log probabilities.
40 # Compute the probabilities.
41 probs = torch.softmax(logits, dim=-1, dtype=torch.float)
42 # Compute the log probabilities (before applying top-p).
42 - logprobs = torch.log(probs, out=logits)
43 + logprobs = torch.log(probs)
44
45 # Apply top-p truncation.
46 top_ps = _get_top_ps(input_metadata)